00010000:00000010:0.1:1786173756.394534:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173756.394543:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173756.402546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173756.402552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173756.412199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173756.420566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173756.425538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173756.425547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173756.437532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173756.437538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173756.442534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173756.450600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173756.459533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173756.459541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173756.473548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173756.473555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173756.477547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173756.490533:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173756.496551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173756.496559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173756.496567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173756.508538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173756.508544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173756.512534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173756.529546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173756.529554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173756.529558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173756.537546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173756.537552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173756.537554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173756.545543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173756.554549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173756.554557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173756.567546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173756.567558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173756.567561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173756.571530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173756.575545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173756.575554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173756.583544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173756.583553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173756.583560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173756.595543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173756.600627:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173756.600634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173756.610541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173756.610548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173756.614544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173756.635556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173756.635566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173756.635573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173756.638613:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173756.638619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173756.642621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173756.651537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173756.655543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173756.655549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173756.664532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173756.664539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173756.668537:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173756.679550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173756.683533:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173756.683539:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173756.691542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173756.691549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173756.699535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173756.709539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173756.712547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173756.712553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173756.721534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173756.721540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173756.724556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173756.728543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173756.736540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173756.736547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173756.740527:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173756.740533:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173756.748536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173756.754538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173756.762547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173756.762553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173756.767548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173756.767556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173756.772531:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173756.782540:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173756.787553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173756.787562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173756.795546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173756.795555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173756.799536:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173756.812545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173756.817541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173756.817552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173756.827544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173756.827550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173756.830540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173756.834539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173756.843605:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173756.843613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173756.852535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173756.852541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173756.857557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173756.865561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173756.873551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173756.873561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173756.887544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173756.887558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173756.887565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173756.899540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173756.899549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173756.899557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173756.903541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173756.903547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173756.907546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173756.916535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173756.921543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173756.921549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173756.928537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173756.928543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173756.932533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173756.940566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173756.946535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173756.946542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173756.953535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173756.953540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173756.953544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173756.960531:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173756.969534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173756.969540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173756.975608:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173756.975616:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173756.980538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173756.988532:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173756.994550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173756.994555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173757.000541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173757.000548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173757.008543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173757.012540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173757.020528:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173757.020533:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173757.029531:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173757.029537:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173757.037538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173757.041535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173757.049547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173757.049555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173757.057540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173757.057546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173757.061543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173757.069551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173757.073546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173757.073556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173757.082539:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173757.082547:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173757.086536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173757.094533:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173757.102553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173757.102562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173757.115544:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173757.115551:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173757.122541:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173757.130547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173757.138058:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173757.138067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173757.146530:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173757.146537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173757.158532:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173757.170543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173757.170560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173757.179531:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173757.196525:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173757.196532:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173757.208547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173757.224553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173757.224561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173757.236534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173757.249542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173757.249551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173757.265529:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173757.273545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173757.273555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173757.285540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173757.297549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173757.297555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173757.309540:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173757.322554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173757.322562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173757.334532:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173757.350559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173757.350570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173757.366564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173757.378528:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173757.378535:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173757.394543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173757.408551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173757.408557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173757.424541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173757.440541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173757.440548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173757.455549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173757.470544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173757.470552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173757.486606:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173757.499602:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173757.499608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173757.507543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173757.507551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173757.524547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173757.536535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173757.536543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173757.549541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173757.561548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173757.561555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173757.581532:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173757.589541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173757.589547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173757.600533:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173757.618539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173757.618547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173757.630551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173757.642536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173757.642543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173757.657542:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173757.669546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173757.669555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173757.685526:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173757.699558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173757.699565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173757.715534:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173757.733554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173757.733561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173757.752527:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173757.764545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173757.764554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173757.783535:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173757.795540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173757.795547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173757.807530:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173757.815541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173757.815549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173757.827535:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173757.840543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173757.840549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173757.848537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173757.861602:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173757.861608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173757.871549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173757.882559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173757.882568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173757.894530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173757.905545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173757.905551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173757.916549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173757.933547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173757.933562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173757.941548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173757.950542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173757.950551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173757.963554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173757.978618:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173757.978633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173757.991539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173758.004535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173758.004544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173758.020547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173758.034538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173758.034545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173758.045550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173758.061549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173758.061558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173758.075548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173758.091547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173758.091556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173758.103550:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173758.118534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173758.118538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173758.127544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173758.139541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173758.139548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173758.153542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173758.165539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173758.165546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173758.178532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173758.197545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173758.197553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173758.203549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173758.215544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173758.215555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173758.230549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173758.246544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173758.246554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173758.258531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173758.284543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173758.284552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173758.299550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173758.311545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173758.311552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173758.329552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173758.338544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173758.338554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173758.351552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173758.363544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173758.363552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173758.381550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173758.393539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173758.393548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173758.406551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173758.419545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173758.419551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173758.434546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173758.452541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173758.452547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173758.463537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173758.476541:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173758.476551:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173758.494549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173758.516544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173758.516552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173758.525532:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173758.539549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173758.547544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173758.547553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173758.559538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173758.572550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173758.572562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173758.586539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173758.601560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173758.601571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173758.617559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173758.631556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173758.631566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173758.647547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173758.663552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173758.663558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173758.678530:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173758.695547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173758.695555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173758.706542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173758.719553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173758.719564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173758.732555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173758.743548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173758.743555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173758.756542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173758.771538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173758.771549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173758.783530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173758.791533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173758.791538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173758.804536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173758.816548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173758.816556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173758.824537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173758.838575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173758.838580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173758.851537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173758.865524:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173758.865530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173758.877552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173758.889556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173758.889562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173758.901541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173758.913528:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173758.913536:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173758.925525:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173758.941545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173758.941553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173758.952578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173758.965547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173758.965556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173758.977540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173758.989530:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173758.989536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173759.001562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173759.017539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173759.017545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173759.029541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173759.040550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173759.040559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173759.059547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173759.067553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173759.067563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173759.075538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173759.088539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173759.088548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173759.100525:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173759.116542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173759.116551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173759.128533:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173759.141529:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173759.141535:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173759.154540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173759.164529:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173759.164537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173759.177548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173759.189557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173759.189564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173759.206534:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173759.218545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173759.218554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173759.230565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173759.242541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173759.242547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173759.258544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173759.274548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173759.274555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173759.287540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173759.299530:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173759.299536:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173759.312530:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173759.326542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173759.326549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173759.340546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173759.350610:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173759.350615:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173759.363531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173759.379536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173759.379541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173759.387535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173759.400548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173759.400557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173759.412540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173759.424543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173759.424550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173759.436538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173759.448543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173759.448553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173759.460547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173759.472539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173759.472547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173759.482532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173759.498523:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173759.498530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173759.506548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173759.518591:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173759.518596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173759.535539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173759.550544:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173759.550552:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173759.564548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173759.564558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173759.572543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173759.584533:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173759.584541:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173759.596539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173759.612537:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173759.612547:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173759.630539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173759.638546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173759.638554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173759.646524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173759.654542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173759.654552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173759.669604:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173759.682524:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173759.682528:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173759.695536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173759.711539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173759.711550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173759.724554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173759.740536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173759.740545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173759.752546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173759.768535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173759.768541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173759.780556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173759.792553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173759.792562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173759.804527:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173759.819595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173759.819602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173759.834539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173759.852549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173759.852554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173759.865547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173759.880576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173759.880582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173759.893613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173759.913589:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173759.913595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173759.926546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173759.942543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173759.942548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173759.954532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173759.966536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173759.966545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173759.982538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173759.995543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173759.995549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173760.012528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173760.025544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173760.025554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173760.037538:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173760.050525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173760.050532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173760.067542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173760.079572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173760.079580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173760.092561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173760.105538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173760.105544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173760.117535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173760.131897:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173760.131904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173760.148532:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173760.160563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173760.160571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173760.176541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173760.188544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173760.188551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173760.201539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173760.211535:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173760.211546:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173760.227549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173760.239536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173760.239544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173760.251545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173760.267533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173760.267538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173760.279539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173760.292545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173760.292554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173760.304546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173760.318533:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173760.318538:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173760.329545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173760.342549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173760.342558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173760.354529:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173760.370561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173760.370571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173760.386555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173760.399535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173760.399541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173760.411537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173760.424545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173760.424552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173760.437551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173760.453548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173760.453555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173760.465534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173760.477528:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173760.477534:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173760.490547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173760.506534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173760.506543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173760.517859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173760.532547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173760.532554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173760.544543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173760.556555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173760.556563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173760.568664:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173760.577552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173760.589551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173760.589560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173760.607533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173760.621534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173760.621541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173760.637554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173760.649538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173760.649547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173760.661536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173760.672552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173760.672562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173760.688539:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173760.701533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173760.701538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173760.713541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173760.725547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173760.725555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173760.737543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173760.754540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173760.754549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173760.762596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173760.774527:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173760.774534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173760.790545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173760.806529:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173760.806537:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173760.823545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173760.835545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173760.835551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173760.848540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173760.860549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173760.860555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173760.876592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173760.892544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173760.892554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173760.915550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173760.928544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173760.928552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173760.944555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173760.956552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173760.956560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173760.968584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173760.988239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173760.988247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173761.006540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173761.020532:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173761.020543:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173761.036542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173761.049540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173761.049546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173761.062533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173761.077548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173761.077555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173761.089533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173761.105544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173761.105553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173761.121546:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173761.132591:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173761.132598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173761.149547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173761.165544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173761.165550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173761.190544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173761.207433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173761.207441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173761.223540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173761.244537:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173761.244544:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173761.262697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173761.286811:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173761.286819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173761.303561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173761.320557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173761.320563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173761.333580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173761.367364:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173761.391026:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173761.391035:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173761.406540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173761.422541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173761.422551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173761.434538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173761.447555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173761.447564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173761.462539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173761.478538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173761.478544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173761.494545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173761.506538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173761.506545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173761.520533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173761.530541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173761.530547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173761.543540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173761.555527:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173761.555532:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173761.564547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173761.577540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173761.577547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173761.591557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173761.603545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173761.619558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173761.619572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173761.635534:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173761.647545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173761.647552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173761.663543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173761.675543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173761.675550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173761.696536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173761.708548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173761.708557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173761.720536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173761.737538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173761.737547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173761.746532:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173761.758536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173761.758543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173761.771534:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173761.791543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173761.791553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173761.804540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173761.820544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173761.820552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173761.831540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173761.849544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173761.849553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173761.861558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173761.874533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173761.874540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173761.886534:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173761.899552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173761.899559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173761.914528:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173761.931550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173761.931560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173761.943560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173761.956573:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173761.956580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173761.971533:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:0.1:1786173761.980545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173761.989554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173761.989563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173762.006905:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173762.019548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173762.019555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173762.035532:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173762.045549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173762.045556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173762.058547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173762.070535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173762.070544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173762.082524:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173762.096544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173762.096551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173762.111541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173762.135537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173762.135544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173762.153532:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173762.158550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173762.158555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173762.170532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173762.183537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173762.183543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173762.195539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173762.206533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173762.206543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173762.218550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173762.231553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173762.231562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173762.245548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173762.261546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173762.261554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173762.274584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173762.287547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173762.287557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173762.300544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173762.316541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173762.316551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173762.328535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173762.344569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173762.344575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173762.357536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173762.371542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173762.371549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173762.383531:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173762.391549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173762.391558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173762.405558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173762.417540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173762.417546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173762.429548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173762.442545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173762.442550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173762.451543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173762.467543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173762.467549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173762.475535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173762.488525:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173762.488530:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173762.501550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173762.512527:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173762.512535:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173762.524549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173762.537532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173762.537538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173762.550545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173762.565535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173762.565539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173762.577528:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173762.585524:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173762.585529:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173762.594532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173762.609540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173762.609556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173762.618545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173762.629533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173762.641548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173762.641554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173762.657582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173762.673538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173762.673543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173762.683532:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173762.699546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173762.699553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173762.705542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173762.720547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173762.720556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173762.731721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173762.746526:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173762.746532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173762.761553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173762.775543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173762.775552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173762.787576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173762.803547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173762.803560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173762.817545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173762.829542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173762.829552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173762.841534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173762.856546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173762.856553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173762.877546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173762.893549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173762.893556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173762.900538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173762.915541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173762.915548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173762.926542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173762.938553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173762.938561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173762.951540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173762.964546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173762.964552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173762.978547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173762.985533:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173762.985542:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173762.999556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173763.012541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173763.012548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173763.024543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173763.034528:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173763.034533:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173763.047551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173763.059539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173763.059552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173763.070579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173763.082544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173763.082553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173763.093571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173763.107529:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173763.107536:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173763.123546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173763.134535:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173763.134542:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173763.147545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173763.163604:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173763.163613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173763.176548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173763.187544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173763.187551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173763.200542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173763.213542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173763.213550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173763.224547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173763.236556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173763.236562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173763.247542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173763.261550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173763.261557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173763.276547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173763.288544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173763.288550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173763.299540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173763.314537:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173763.314556:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173763.326548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173763.337551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173763.337559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173763.348545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173763.363536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173763.363542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173763.378540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173763.389548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173763.389556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173763.399537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173763.414542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173763.414552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173763.421548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173763.436542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173763.436550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173763.447550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173763.458539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173763.458545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173763.469538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173763.480549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173763.480556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173763.490525:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173763.502526:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173763.502532:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173763.516542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173763.532544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173763.532551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173763.541536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173763.553572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173763.553579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:0.1:1786173763.561612:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173763.570529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065d00.
00000080:00000010:0.1:1786173763.574745:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0b40.
00010000:00000010:0.1:1786173763.579539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065e00.
00000080:00000010:0.1:1786173763.584535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0ac0.
00010000:00000010:0.1:1786173763.588531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065d00.
00000080:00000010:0.1:1786173763.592534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0b40.
00010000:00000010:0.1:1786173763.597552:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065900.
00000080:00000010:0.1:1786173763.600532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0e60.
00010000:00000010:0.1:1786173763.605539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065300.
00000080:00000010:0.1:1786173763.614535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0b40.
00010000:00000010:0.1:1786173763.614548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065700.
00000080:00000010:0.1:1786173763.621545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0e60.
00010000:00000010:0.1:1786173763.621562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065a00.
00000080:00000010:0.1:1786173763.629556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0ac0.
00010000:00000010:0.1:1786173763.638550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065a00.
00000080:00000010:0.1:1786173763.642544:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0e60.
00010000:00000010:0.1:1786173763.642560:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065100.
00000080:00000010:0.1:1786173763.650534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0b40.
00010000:00000010:0.1:1786173763.661554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065800.
00000080:00000010:0.1:1786173763.661559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0ac0.
00010000:00000010:0.1:1786173763.668543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065a00.
00000080:00000010:0.1:1786173763.668553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0e60.
00010000:00000010:0.1:1786173763.673545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065c00.
00000080:00000010:0.1:1786173763.677529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0b40.
00010000:00000010:0.1:1786173763.681546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065a00.
00000080:00000010:0.1:1786173763.689535:0:2384:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0ac0.
00010000:00000010:0.1:1786173763.691546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065c00.
00000080:00000010:0.1:1786173763.696543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0e60.
00010000:00000010:0.1:1786173763.696552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19800.
00010000:00000010:0.1:1786173763.708544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173763.708552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173763.720545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173763.734535:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173763.734543:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173763.753590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173763.766542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173763.766551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173763.778553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173763.797551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173763.797560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173763.806535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173763.824537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173763.824544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173763.836543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173763.845540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173763.845548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173763.857527:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173763.872543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173763.872549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173763.885528:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173763.894539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173763.894549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173763.908534:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173763.914543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173763.914551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173763.935525:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173763.941541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173763.941548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173763.960546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173763.973562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173763.973570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173763.989527:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173764.000544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173764.000551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173764.022525:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173764.033997:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173764.034007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173764.055549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173764.072547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173764.072557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173764.086101:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173764.111378:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173764.111384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173764.131588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173764.153669:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173764.153675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173764.173544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173764.191651:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173764.191660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173764.219533:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5e400.
00010000:00000010:0.1:1786173764.229535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9be00.
00010000:00000010:0.1:1786173764.229545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5e200.
00010000:00000010:0.1:1786173764.242534:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173764.251547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173764.251553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173764.266571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173764.278546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173764.278552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173764.291538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173764.308547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173764.308553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173764.316536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173764.332523:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173764.332527:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173764.344545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173764.363551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173764.363560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173764.380532:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173764.388541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173764.388549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173764.400544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173764.413550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173764.413559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173764.425547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173764.440570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173764.440575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173764.453527:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173764.463540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173764.463547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173764.476553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173764.488528:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173764.488534:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173764.506536:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173764.522534:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173764.522540:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173764.531542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173764.543560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173764.543568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173764.556535:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173764.572526:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173764.572531:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173764.585556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173764.601540:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173764.601548:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173764.614549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173764.622545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173764.622552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173764.641544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173764.655544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173764.655552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173764.671542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173764.690583:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173764.690589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:3.1:1786173764.696560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173764.701548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:3.1:1786173764.709538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173764.714577:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173764.714582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:3.1:1786173764.720572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:3.1:1786173764.720578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173764.729525:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:3.1:1786173764.732544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173764.736550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173764.736556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:3.1:1786173764.743548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:3.1:1786173764.743558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173764.747530:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:3.1:1786173764.757550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173764.760551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173764.760558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:3.1:1786173764.775544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:3.1:1786173764.775553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173764.777550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:3.1:1786173764.783566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173764.786548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173764.786557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:3.1:1786173764.799546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:3.1:1786173764.799556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173764.801542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:3.1:1786173764.810536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173764.815533:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173764.815539:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:3.1:1786173764.825548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:3.1:1786173764.825554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173764.829546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173764.842547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173764.842553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173764.858535:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173764.870983:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173764.870989:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173764.892607:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173764.909550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173764.909558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173764.921550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173764.938545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173764.938555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173764.951543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173764.968563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173764.968570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173764.984576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173764.999549:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173764.999563:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173765.011544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173765.026548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173765.026559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173765.041542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173765.053551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173765.053559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173765.066536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173765.083559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173765.083567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173765.100549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173765.107543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173765.107551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173765.119538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173765.139542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173765.139551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173765.152534:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173765.165541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173765.165547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173765.186546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173765.190540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173765.190545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173765.204544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173765.216542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173765.216550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173765.233540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173765.245551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173765.245558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173765.251577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173765.263537:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173765.263543:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173765.275542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173765.287537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173765.287544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173765.299537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173765.312552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173765.312562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173765.324553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173765.336531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173765.336538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173765.348543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173765.360536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173765.360542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173765.373543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173765.385543:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173765.385548:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173765.397549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173765.410527:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173765.410532:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173765.422533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173765.435551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173765.435559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173765.452527:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173765.461554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173765.461565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173765.481592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173765.494550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173765.494559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173765.506642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173765.526609:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173765.526617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173765.545547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173765.557621:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173765.557628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173765.574547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173765.591548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173765.591556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173765.605561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173765.620541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173765.620547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173765.633554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173765.650547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173765.650558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173765.666532:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173765.685562:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173765.685572:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173765.698541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173765.703565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173765.717543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173765.717554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173765.729547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173765.741533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173765.741539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173765.753544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173765.770525:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173765.770530:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173765.783558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173765.799558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173765.799567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173765.813547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173765.828546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173765.828556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173765.844565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173765.857534:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173765.857541:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173765.873548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173765.886541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173765.886548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173765.897586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173765.914548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173765.914556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173765.929607:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173765.944547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173765.944554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173765.955545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173765.972544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173765.972552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173765.988553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173766.004551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173766.004559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173766.022547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173766.033594:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173766.033598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173766.048552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173766.056558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173766.056566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173766.068540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173766.080538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173766.080545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173766.089526:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173766.104547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173766.104552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173766.116534:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173766.130540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173766.130547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173766.143616:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173766.164549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173766.164554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173766.176542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173766.190530:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173766.190538:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173766.202552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173766.220545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173766.220552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173766.239540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173766.244544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173766.244552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173766.253541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173766.272530:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173766.272536:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173766.286578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173766.294535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173766.294544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173766.306542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173766.324578:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173766.324586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173766.332550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173766.345546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173766.345555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173766.353530:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173766.368546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173766.368556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173766.380549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173766.393548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173766.393556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173766.405544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173766.422551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173766.422563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173766.438550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173766.456559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173766.456567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173766.472548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173766.488543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173766.488552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173766.500596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173766.517537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173766.517547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173766.529538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173766.541553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173766.541561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173766.553556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173766.561556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173766.561565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173766.574544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173766.586535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173766.586540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173766.598550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173766.610528:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173766.610537:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173766.622537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173766.635550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173766.635559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173766.647543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173766.662548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173766.662561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173766.670538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173766.684536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173766.684541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173766.700549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173766.713536:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173766.713542:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173766.733551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173766.745546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173766.745553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173766.758546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173766.774551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173766.774557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173766.786545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173766.802545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173766.802551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173766.814587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173766.830538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173766.830546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173766.843545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173766.857522:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173766.857529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173766.872547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173766.884546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173766.884553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173766.894680:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173766.910556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173766.910563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173766.923560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173766.939541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173766.939546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173766.951536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173766.962617:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173766.962626:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173766.975547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173766.987527:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173766.987533:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173767.000535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173767.012556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173767.012564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173767.029549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173767.045536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173767.045543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173767.055552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173767.071778:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173767.071784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173767.087534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173767.101561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173767.101570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173767.114541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173767.130543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173767.130552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173767.142536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173767.154538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173767.154547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173767.161529:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173767.172535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173767.172540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173767.185534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173767.197540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173767.197546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173767.209537:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173767.219533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173767.219540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173767.231532:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173767.243544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173767.243554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173767.256532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173767.268548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173767.268555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173767.284550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173767.296539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173767.296544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173767.308542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173767.324527:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173767.324535:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173767.336542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173767.352534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173767.352539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173767.364536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173767.378534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173767.378540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173767.395542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173767.406546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173767.406560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173767.418553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173767.430532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173767.430538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173767.442538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173767.455534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173767.455541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173767.467551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173767.479544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173767.479553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173767.491541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173767.503552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173767.503562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173767.519541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173767.531530:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173767.531535:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173767.543552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173767.553586:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173767.553592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173767.568536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173767.580539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173767.580544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173767.592539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173767.604540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173767.604551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173767.616530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173767.629532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173767.629538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173767.641546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173767.656542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173767.656550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173767.672553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173767.685625:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173767.685634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173767.702540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173767.718544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173767.718553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173767.730550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173767.742549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173767.742558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173767.750546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173767.750554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173767.763552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173767.776543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173767.776551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173767.788546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173767.804544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173767.804552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173767.812550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173767.828533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173767.828538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173767.840533:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173767.852538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173767.852545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173767.861546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173767.877543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173767.877554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173767.889543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173767.901540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173767.901550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173767.913547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:0.1:1786173767.927544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173767.927549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173767.937535:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173767.949546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173767.949554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173767.961538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:0.1:1786173767.973548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173767.973553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:0.1:1786173767.990547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:0.1:1786173768.006537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173768.006543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:0.1:1786173768.022536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:0.1:1786173768.034526:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173768.034532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:0.1:1786173768.050526:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:0.1:1786173768.062543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173768.062551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:0.1:1786173768.078540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:0.1:1786173768.090548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173768.090555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173768.105534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173768.121548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173768.121556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:0.1:1786173768.134536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173768.145541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173768.145548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173768.158541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173768.171544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173768.171553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:0.1:1786173768.184550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:0.1:1786173768.196544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173768.196553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:0.1:1786173768.208531:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173768.216534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173768.220529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173768.221544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173768.221551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173768.224548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173768.228539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.232531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173768.232545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173768.236552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.238538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173768.238546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173768.238554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173768.241546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173768.245536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173768.250550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173768.254537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173768.258531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173768.262530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.266542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173768.267537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173768.267545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173768.270553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173768.275562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173768.279540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173768.280535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173768.283538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173768.287541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.291531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173768.291537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173768.291542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173768.295543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173768.299542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173768.303527:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173768.303543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.307553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173768.312542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.316538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173768.316541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173768.316550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173768.320551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173768.325548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173768.329534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173768.329551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173768.333546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173768.337537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.341536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173768.345541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173768.346539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173768.346549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173768.349532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173768.357558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173768.357560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.357573:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173768.361547:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.366570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173768.370549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173768.370561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173768.374544:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173768.378544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173768.387537:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173768.387546:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.391550:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173768.395532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.399546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173768.403542:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173768.403551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173768.403561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173768.408548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173768.414554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173768.418538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173768.422543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173768.426544:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.431557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173768.435563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173768.435570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173768.443536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173768.447529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173768.448532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173768.451531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.451541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173768.455538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.459537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173768.463528:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173768.463534:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173768.463539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.468532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173768.475536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173768.475543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173768.475551:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173768.483531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.483540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173768.487533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173768.487551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173768.487560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173768.491530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173768.494568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.499534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173768.499538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173768.504535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.508546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173768.512536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173768.512552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173768.512559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173768.516540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173768.520532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173768.520541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.524564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173768.528537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.532537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173768.536539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173768.536542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173768.536550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173768.540535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173768.544537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173768.548544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173768.548554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173768.552555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.554536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173768.554546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173768.554554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173768.558535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.563532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173768.569539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.569546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173768.572539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173768.576534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.580548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173768.584532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173768.584537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173768.584544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173768.588533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173768.592530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.596533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173768.596540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173768.600543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.603622:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173768.607684:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173768.607689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173768.608529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.617549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173768.621544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173768.621546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.625560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173768.630531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.634549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173768.638539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173768.638546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173768.642538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.646535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173768.650533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173768.651531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173768.654543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173768.659549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.663533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173768.663590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173768.663595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173768.667534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.671534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173768.675543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173768.679535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173768.679543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173768.684528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.689544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173768.692526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173768.692541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173768.692550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173768.696529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173768.698526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173768.702535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173768.702546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173768.710554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.714530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173768.714540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173768.714549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173768.718553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.718564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173768.726536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173768.726548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.726556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173768.731537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.735607:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173768.740531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173768.740551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173768.740561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173768.748537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173768.752539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173768.752551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173768.758529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173768.761550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.765534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173768.765540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173768.769552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173768.769558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173768.769564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173768.777532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173768.777535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173768.777542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173768.781529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173768.785551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.789548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173768.789550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173768.797549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.797560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173768.801541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173768.801549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173768.805550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.809543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173768.813526:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173768.814541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.817545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173768.823535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.823544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173768.826545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173768.826553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173768.830537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.830544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173768.838536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.838549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173768.838562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173768.843527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.847532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173768.851536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173768.851547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173768.851556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173768.855536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173768.859548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173768.863539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173768.863546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173768.869533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.873532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173768.873541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173768.873549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173768.877536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.881535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173768.885537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.889549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173768.890542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173768.897534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173768.901543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:2.1:1786173768.901546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173768.901550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173768.906531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.910547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173768.914532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173768.914536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.918532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173768.922529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.926534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173768.930547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173768.930554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173768.930562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173768.934534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173768.938546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.941596:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173768.942538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173768.946545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.954531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173768.954545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173768.954552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173768.958537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.958545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173768.966538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173768.966543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173768.966552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173768.972529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173768.976532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173768.977527:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173768.977534:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173768.980546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173768.984528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173768.988532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173768.992531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173768.992535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173768.998551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.000544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173769.004537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173769.004545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173769.005537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.005544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173769.012536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.013534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173769.016531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173769.020544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173769.024533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173769.024538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173769.024542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173769.028536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.028545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173769.036530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.036536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173769.036543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173769.045539:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.045555:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173769.047540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.047544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173769.047553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173769.051542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173769.060533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.060541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173769.064541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173769.068575:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173769.068582:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173769.073529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.073552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173769.073568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173769.077527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173769.085537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.090532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173769.090538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173769.099552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.099562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173769.103547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173769.103549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173769.103556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173769.108552:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173769.108561:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173769.111541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173769.111548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173769.116538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.125531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173769.125541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173769.125551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173769.130539:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.130551:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173769.138551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.142533:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173769.142537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173769.146536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.146547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173769.150550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173769.150559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173769.154538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173769.154550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173769.158547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.162552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173769.166529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173769.170555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.174550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173769.178537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173769.178551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173769.178561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173769.186546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173769.194535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.194539:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173769.194550:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173769.198536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173769.202548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173769.206530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.206545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173769.206554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173769.210536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173769.218534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173769.218542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173769.222531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173769.226547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.226566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173769.231542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173769.231549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173769.234528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.240537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173769.243533:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173769.243533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173769.247537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173769.255547:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.259543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173769.264570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.264902:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173769.264925:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173769.267553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173769.272551:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173769.276537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173769.280545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173769.284535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.288531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173769.292531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.292552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173769.292562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173769.296529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173769.300530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173769.304542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173769.304550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173769.313550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.317546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173769.317553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173769.318531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173769.321525:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.326543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173769.330533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173769.330540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173769.334531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173769.338536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.342536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173769.342538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173769.342544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173769.346541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.350548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173769.354524:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173769.354544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173769.358538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173769.362544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.366537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173769.370533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.370546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173769.370554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173769.374537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173769.378543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173769.382529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173769.382546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173769.386544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.390544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173769.393585:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173769.393591:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173769.401535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173769.401549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173769.406545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.406555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173769.406561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173769.411539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.415533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173769.420549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173769.420563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173769.427540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.431541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173769.435532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173769.435546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.435562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173769.440542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.444548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173769.451542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173769.451549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173769.456538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.464548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173769.469539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173769.470532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.470543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173769.474534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.481597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173769.481602:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173769.482529:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173769.482534:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173769.486547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173769.490527:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173769.490542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.494542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173769.498532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.502528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173769.502548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173769.502554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173769.506535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.510534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173769.514553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.518528:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173769.518546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173769.522540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.526544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173769.531544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173769.531551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173769.531558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.535556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173769.540560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173769.544544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173769.544552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173769.549543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.553546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173769.557544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173769.557553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173769.561551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.566545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173769.570536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.570543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173769.575542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173769.580547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.584539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173769.584547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173769.584551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173769.588545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.593531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173769.596537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.596551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173769.600531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173769.604538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.608534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173769.608539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173769.608549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173769.612540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.616543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173769.620535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.621575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173769.624533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173769.628561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.632553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173769.636538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173769.636544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173769.636551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173769.640542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173769.649542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.650540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173769.653533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173769.657533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.661531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173769.665533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.665544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173769.665554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173769.669533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173769.673540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.677536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173769.677540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173769.684609:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.684617:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173769.689545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173769.689553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173769.693552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.698540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173769.702548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173769.702602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173769.706543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173769.712553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.716547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173769.716549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173769.716555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173769.720537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.724563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173769.728547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173769.732550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173769.732553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173769.737540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.742539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173769.745537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173769.745543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173769.749531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.749538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173769.758529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.767545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173769.767552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173769.771534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.775536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173769.775591:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173769.775600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173769.779562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.783545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173769.791550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173769.791586:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.791598:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173769.801536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.805539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173769.805546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173769.809598:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173769.809606:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173769.813536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173769.817587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.825532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173769.825549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173769.834545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.838541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173769.838556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173769.838563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173769.842536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.846532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173769.850538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.854542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173769.854567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173769.862535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.866545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173769.870528:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173769.870535:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173769.870544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.874535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173769.882574:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.885580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173769.886544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173769.894575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.899547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173769.902588:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173769.902594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173769.906531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.906548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173769.912539:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173769.920536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173769.920557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173769.925541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173769.930542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173769.934538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173769.934544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173769.938545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.943548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173769.947534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173769.947546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173769.951532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173769.956528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173769.960529:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:2.1:1786173769.960531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173769.960536:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173769.964531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.968527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173769.972540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173769.976530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173769.976537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173769.980528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173769.984545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173769.988540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173769.988546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173769.988552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173769.992537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173769.996537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173770.000532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173770.000537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173770.004539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.008537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173770.012529:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173770.012535:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173770.012541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.016533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173770.020544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.024538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173770.024546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173770.028535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.032544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173770.036534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173770.036536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173770.036544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173770.040543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173770.044542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.048538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173770.049532:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173770.052539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173770.056547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:2.1:1786173770.056549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173770.056556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173770.060537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.064543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173770.068541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173770.068542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.073534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173770.076542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.080543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173770.084539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173770.084546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173770.084548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.088539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173770.092542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173770.092547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173770.096552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173770.100532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.104532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173770.106536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173770.106556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173770.106566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173770.110557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173770.115549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173770.121546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173770.123532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173770.127548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.131569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173770.134548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173770.134557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173770.138546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.142536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173770.149574:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.149584:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173770.150562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173770.158538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.158547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173770.162541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173770.162550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173770.165580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.165590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173770.173541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173770.173550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173770.176551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173770.181536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.185544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173770.193635:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173770.193644:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173770.198548:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.198566:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173770.202547:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173770.202548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173770.208545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173770.212536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.216535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173770.216545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173770.216554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173770.220533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.228538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173770.232543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173770.232547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173770.236532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173770.240539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.240550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173770.245535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173770.245543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173770.250534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.250545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173770.254532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173770.258535:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173770.258547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173770.262539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.270531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173770.279530:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173770.279537:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173770.283539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.283547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173770.287534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.287547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173770.288536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173770.291539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.291548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173770.297539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173770.297543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173770.297550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173770.302540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173770.306535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173770.307536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.310538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173770.314541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.314553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173770.319540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173770.319545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173770.323529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.327532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173770.331531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.331539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173770.331555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173770.335531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.344550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173770.348553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.350528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173770.350534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173770.350544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173770.350553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173770.354546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173770.356541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173770.358541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.362543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173770.370541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173770.370547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173770.371532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.371540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173770.379540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173770.379547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.379560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173770.388547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.392535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173770.394551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173770.394557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173770.399532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.399545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173770.403540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173770.407533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.407543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173770.411529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.415535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173770.419536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173770.419546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173770.419554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173770.422543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173770.426530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173770.430545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173770.434585:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173770.438531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.442532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173770.447532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173770.447536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173770.447543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173770.451531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173770.455528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.459529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173770.459536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173770.463550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.467535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173770.471539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173770.471546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173770.475550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.475562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173770.480540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.484552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173770.485535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173770.492545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.492557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173770.496539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173770.496546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173770.500532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.504539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173770.508527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.512532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173770.512533:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173770.516529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.520533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173770.524534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173770.524542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173770.524551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173770.528531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173770.532533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.536535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173770.536543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173770.540535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.544539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173770.548530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173770.548536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173770.548544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173770.552546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173770.556529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.560532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173770.560538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173770.564534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.568536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173770.572530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173770.573541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173770.573548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173770.576534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173770.580532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.584537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173770.584542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173770.588536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.592533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173770.596543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173770.596549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173770.596557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173770.600545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173770.605546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173770.609533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173770.609547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173770.620548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.620560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173770.624537:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173770.624545:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173770.624547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.627533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173770.631551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173770.635536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173770.635540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173770.639529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.643535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173770.647535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173770.647540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173770.647551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.651531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173770.655541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173770.659528:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173770.660539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173770.664552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.668545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173770.672530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173770.672552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173770.672562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173770.676532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173770.680536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.684533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173770.684544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173770.689539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.691574:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173770.696539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173770.696548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173770.697543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.700539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173770.708540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.708551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173770.708557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173770.713548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.713557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173770.717552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173770.717559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173770.721530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.721538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173770.725548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.729539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173770.729541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173770.733532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.737536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173770.741538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173770.741552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173770.741559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173770.745538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173770.753529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173770.753555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173770.757547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173770.761546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.761556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173770.765542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173770.765549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173770.766532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.766547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173770.773532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.773549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173770.774524:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173770.777532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.781552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173770.785543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173770.785548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173770.785555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173770.789534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173770.793537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.797526:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173770.797532:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173770.801539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.805537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173770.805545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173770.805550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173770.809528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.809535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173770.817530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.817539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173770.821532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173770.825541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173770.825544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173770.829548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.833534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173770.837537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173770.837542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173770.837543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173770.841529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173770.845547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173770.845549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.849535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173770.853532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.853545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173770.857544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173770.857550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173770.858530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.861536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173770.865532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173770.869530:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173770.869535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173770.873541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.877532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173770.881540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173770.881546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173770.885531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173770.885542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173770.889531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.893538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173770.893547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173770.898534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.902542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173770.906532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173770.906537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173770.906543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.910543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173770.914548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173770.918545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173770.918551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173770.922534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.926555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173770.930526:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173770.930531:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173770.930548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.934537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173770.938531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173770.938538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173770.942542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173770.947538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.951527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173770.951549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173770.951556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173770.955545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.959547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173770.963543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173770.963546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173770.967537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173770.971539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.975534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173770.975543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173770.975553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173770.979535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173770.979542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173770.986574:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173770.986582:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173770.987538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173770.991527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173770.995528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173770.999532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173770.999540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173770.999548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173771.003527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173771.007531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173771.011536:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173771.012545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173771.015535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.019536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173771.019544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173771.019552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173771.023534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.033534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173771.040535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173771.040542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:0.1:1786173771.048527:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173771.048533:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173771.048542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173771.048549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.048560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173771.052537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173771.057533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173771.057545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173771.057553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.061536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173771.065533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.065544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173771.073547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173771.073550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173771.073554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173771.082536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173771.086535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.086545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173771.094531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.094545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173771.095533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173771.098539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173771.103537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:2.1:1786173771.103547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173771.103548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173771.108533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173771.117538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173771.117545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173771.122551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.122561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173771.127533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173771.127538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173771.127552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.127567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173771.136536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.140531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173771.140541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173771.144534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173771.144535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173771.144543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173771.144547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173771.144563:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.148527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173771.152527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.155637:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173771.156557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173771.163547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173771.165552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173771.165562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173771.169534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173771.169547:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.173529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173771.177547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173771.181533:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173771.183529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173771.186551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.190537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173771.194576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173771.194633:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173771.194640:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173771.199586:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173771.201535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173771.201556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173771.205546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173771.213546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.213559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173771.217532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173771.217538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173771.221533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.225533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173771.229533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.233534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173771.233546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173771.237531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.241532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173771.245531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173771.245544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173771.245551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173771.248541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173771.253544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.257541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173771.257549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173771.266527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173771.273530:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173771.273536:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173771.273548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173771.273554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.273562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173771.281528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.281537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173771.281548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173771.289534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.289544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173771.293535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173771.293634:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173771.293643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173771.297560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173771.301534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.305563:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173771.308590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173771.313543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.317545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173771.321537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173771.321544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173771.325603:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.325613:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173771.330527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.339528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173771.339533:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173771.343534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.352535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173771.352539:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173771.352542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173771.352549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173771.357548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173771.360528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.360536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173771.360543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173771.364543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.364553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173771.368528:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173771.368535:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173771.372539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.372548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173771.380529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.380537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173771.380546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173771.384536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.388530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173771.392533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173771.392538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173771.392545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173771.396528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173771.400545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.404528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173771.404549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173771.408554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.412551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173771.416540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173771.416545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173771.420556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.420566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173771.424543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173771.428537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173771.432549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173771.438541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173771.442537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173771.442545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173771.443551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173771.446537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.446544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173771.454530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173771.454540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173771.458538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173771.463548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.467535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173771.467543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173771.467551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173771.471530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.475553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173771.479544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173771.483534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173771.483541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173771.487538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.491536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173771.495551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173771.495559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173771.499541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.503531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173771.507546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173771.507550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173771.513545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173771.518532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173771.521542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:2.1:1786173771.521548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173771.521550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173771.526549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.526566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173771.533542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173771.533545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.533561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173771.538531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.542545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173771.546531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173771.546536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173771.550532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.555534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173771.559526:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173771.560542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.563545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173771.568544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.572531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173771.572547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173771.572557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173771.576556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.576570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173771.581526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.585537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173771.585541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173771.589530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.593543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173771.597543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173771.597563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173771.597570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173771.599536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173771.607535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173771.608546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.612534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173771.616533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.620537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173771.620539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173771.620547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173771.625534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.629532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173771.634530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.638529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173771.638532:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173771.642528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.646531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173771.651533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173771.651542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173771.651550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173771.655531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173771.659562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.664532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173771.669528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173771.669532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173771.674538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173771.681534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173771.681538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173771.681546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173771.686542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173771.691540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173771.691545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173771.691555:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173771.701536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.706533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173771.711534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173771.711536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173771.711544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173771.711546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173771.715529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.715538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173771.715544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173771.720535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.724533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173771.724548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173771.724556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173771.728543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.733534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173771.737540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173771.737590:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.741537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173771.745550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173771.749528:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173771.749534:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173771.749555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173771.753535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.757544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173771.761534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.765535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173771.765538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173771.769533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173771.778534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173771.778536:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173771.778545:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173771.783544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.791538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173771.791546:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173771.791550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173771.795537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173771.799531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.799541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173771.803540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173771.803555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173771.803563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173771.807536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173771.811540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173771.814564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173771.815543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173771.820549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.824612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173771.828542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173771.828548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173771.830539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.834553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173771.839537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.843536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173771.851533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173771.851548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.855536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173771.859546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173771.863539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173771.863546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173771.865535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173771.867528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.871529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173771.875558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.879532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173771.879541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173771.883550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.887551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173771.891539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173771.891546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173771.895556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.895567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173771.903527:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173771.903540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.907536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173771.912538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.916537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173771.916553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173771.916561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173771.920537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.924544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173771.928547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173771.929544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.932561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173771.936551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.936565:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173771.944536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173771.945540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173771.945547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173771.948549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173771.952533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.954545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173771.955545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173771.958547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.964556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173771.967541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173771.967547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173771.971534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173771.971544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173771.980527:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173771.980538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.984545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173771.990543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173771.994533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173771.994541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:0.1:1786173771.994551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173771.999535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173771.999551:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173772.004532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.004547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173772.008547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173772.012536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.017533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173772.022533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173772.022536:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173772.022544:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173772.026534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173772.030549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.034530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173772.034544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173772.043534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.043548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173772.047526:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173772.047533:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173772.047545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.051544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173772.066529:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173772.066533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.066548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173772.074533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.074546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173772.074553:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.078532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173772.083528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.086530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173772.090544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.094548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173772.102544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.102558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173772.110534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.114535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173772.118548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.122535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173772.126555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.135543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173772.140538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.140551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.157538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.161529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173772.161535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.165532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173772.165551:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.169533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173772.173532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.177534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173772.182547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.186534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173772.189533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.193534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173772.201535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.201544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173772.209535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.209549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173772.217531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.217541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.221541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.223532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173772.231534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.235545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173772.239535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.247533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173772.251538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.255564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173772.255572:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.259533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173772.263549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.267547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173772.272557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.276547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173772.282541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.285544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173772.289574:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.301535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173772.305532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.305550:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173772.309545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.312617:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173772.320619:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.325533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.329530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.333532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173772.337545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.341540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173772.346542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.350542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173772.355550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.359566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173772.363534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.371559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173772.379529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.379537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173772.383543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.387534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.391544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.395549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173772.399530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.403542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173772.411536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.415549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173772.419530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.423534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173772.427544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.435543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173772.440548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.440618:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173772.448543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.458537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173772.463524:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.463534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173772.467534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.472524:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173772.477529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.486550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173772.490540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.499523:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173772.499527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.502533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173772.506529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.506540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173772.510531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.514536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173772.518547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.522530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173772.526540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.534541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173772.539531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.539544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173772.543545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.547557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173772.552535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.560533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173772.568533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.568552:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173772.572528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.576532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173772.580527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.588534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.593534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.597543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173772.597554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.601529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.605540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.609533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173772.613544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.613554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.621537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.625531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173772.629535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.633538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173772.637538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.637555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173772.642541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.646533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173772.651532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.655528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173772.659538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.667537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173772.675544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.675559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173772.683530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.687534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173772.691532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.691541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173772.696535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.699527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173772.703530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.706608:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.711542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.719536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173772.724545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.724554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173772.731529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.731537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173772.739531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.743539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173772.747532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.752530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173772.756543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.760538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173772.764541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.768542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173772.772529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.776536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.780539:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.784533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173772.788533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.792536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173772.796533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.800542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173772.805531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.809532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173772.817531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.824544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173772.828544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.828560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173772.832529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.840528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173772.844531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.848542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173772.848549:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.856542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173772.861555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.861567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.867525:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.871526:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173772.875525:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.879530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173772.883539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.887536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173772.892529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.892541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173772.899616:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.905553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173772.913536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.913550:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173772.917529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.922542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.926537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.930542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173772.934537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.939527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.942534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.946538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173772.950589:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.958538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173772.966530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.974535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173772.974542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173772.978538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173772.978545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173772.982548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173772.982553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173772.987538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173772.991527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173772.995545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173772.995551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.000533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173773.004547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.012534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173773.016536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.020531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173773.020542:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.024532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173773.028528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.032527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173773.036533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.040530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173773.044529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.048530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173773.052541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.060535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173773.064545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.068549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.072534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.076533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173773.082539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.090534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.094536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.098542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173773.098551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.102540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.114534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.122562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.122569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.126550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173773.131542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.139538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.144547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.144560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173773.149540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.158533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173773.163549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.163564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173773.175529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.177525:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173773.177530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.182538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173773.191532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.200551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173773.200560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.200574:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173773.205543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.205556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173773.220551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.224541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173773.228529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.232556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.240554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.244535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173773.247532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.255537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.260528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.260540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173773.269531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.273532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173773.281544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.285544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173773.289533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.293528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173773.297542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.301544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173773.306548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.310530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173773.314533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.317584:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173773.322526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.330539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.334543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.338556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173773.343546:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.348554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173773.353533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.357548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173773.373543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.373557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.377531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.387558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173773.387567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.390540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.394535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.394546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173773.398555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.402546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.406906:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.415547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173773.423542:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.423559:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173773.427540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.431603:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173773.436555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.445540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.451552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.451570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173773.456530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.464533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173773.468686:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.472536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173773.480532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.484534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173773.488533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.496543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173773.505532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.509531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.509537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.513541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173773.513549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.518535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173773.523532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.523545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173773.531529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.535525:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173773.539637:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.542574:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.555542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.562541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173773.570535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173773.570547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.574525:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.574537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173773.578532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.578547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173773.582527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.586536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173773.590541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.594533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173773.599555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.604538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173773.609546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.614537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173773.619540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.624561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173773.628556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.636536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173773.648531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.652542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173773.652552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.656532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173773.664545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.664555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173773.672534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.676537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173773.680533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.684534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.688550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.693550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173773.702550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.706551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173773.710531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.714545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173773.718533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.722533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173773.727527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.730543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173773.739542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.743543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173773.747529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.747538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173773.756530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.756541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173773.765526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.765534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173773.768542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.772534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173773.777540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.780534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173773.784532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.788540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.792530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.796557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173773.808534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.812535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173773.812544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.816528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173773.823539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.823549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173773.828533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.832530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.840529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.840539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.844530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.848529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173773.852529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.856531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173773.860529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.864531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173773.868541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.872540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.876566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.888545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.892550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.892567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173773.901529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.905546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.909530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.909545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173773.913534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.917537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.933533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.933542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173773.941537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173773.941549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173773.945528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.945537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173773.951528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173773.954532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.958535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.958545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173773.970528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.970536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.974528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173773.979529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173773.979533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173773.987544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173773.991546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173773.995531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173773.995538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.000524:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173774.004546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.008531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173774.016558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.020535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173774.024530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.024540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173774.029534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.032528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173774.036533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.040536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.044524:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.048532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173774.052534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.056532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173774.060554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.064538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173774.076551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.076563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.084526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.092540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173774.097549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.097562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.100540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.104547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173774.108534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.112544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173774.120550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.124527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173774.128545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.132545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173774.137537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.141539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173774.145544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.149534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.159528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.163534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.171547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.175550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.180549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.184534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173774.197551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.197563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173774.209538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.209551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.213544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.213559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173774.218527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.222528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.226535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.230539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173774.233525:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.237549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173774.242540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.246533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173774.253533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.257538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.261527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.261535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173774.273534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.273550:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.281549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.285536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.289536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.289548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.293550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.298527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.302541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.306525:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173774.310537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.314524:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173774.318529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.322531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173774.326523:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.330537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.334539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.338532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173774.342534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.346527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173774.350534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.355555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173774.363546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.363555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.379532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.383532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173774.383539:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.387543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173774.387551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.391538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.396529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.400531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173774.404542:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.408535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173774.412588:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.416552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173774.421541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.425529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173774.433543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.433556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173774.437535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.441539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173774.449543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.449549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173774.457544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.461534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173774.465533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.465541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.474529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.474537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173774.483533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.483549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.487527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.491533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173774.499545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.502551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173774.506527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.511524:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173774.515548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.519528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173774.523540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.527530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173774.531532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.531543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173774.539527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.539536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173774.543538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.549547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173774.555530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.559528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173774.563530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.563538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.567533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.571531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173774.575527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.579526:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173774.587539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.591539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173774.595546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.595560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173774.599540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.603544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173774.607550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.607562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173774.616530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.616538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173774.624528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.624539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173774.628531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.632537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173774.636534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.644536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.649534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.649544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.653534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.657537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.661533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.665530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173774.669538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.673523:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.677535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.683525:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173774.687570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.691542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.691551:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.695545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173774.699537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.703543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173774.707537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.711536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173774.718555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.722533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173774.726530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.726542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.730530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.734537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173774.738547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.742535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.746530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.750528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173774.754801:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.758544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173774.766531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.774528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173774.778527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.782531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173774.782535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.786539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173774.790536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.794528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173774.798529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.802530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173774.806544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.810533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.814544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.818549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173774.823542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.827543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173774.836532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.836543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173774.844542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.844555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173774.852547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.852555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173774.864543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.868534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173774.872528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173774.872540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173774.876528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.881541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173774.884526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.888531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173774.892532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.897535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.900529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.904532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173774.908535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.912532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173774.916536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.920529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173774.924534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.924543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173774.932571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.932587:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173774.940537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.940546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173774.948527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.948535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173774.952528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.956528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173774.961539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.965532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173774.970535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.974530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173774.978532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173774.978547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173774.982539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173774.986529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173774.994532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173774.994541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173774.998532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.002541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.006538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.010529:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173775.014536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.018546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.022528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.022539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173775.030528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.030538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.034530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.038525:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173775.042525:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.047536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.050536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.054534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.058538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.062539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.066539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.066546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.074542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.074554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.082551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.082564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173775.090545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.090555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173775.100544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.100554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173775.112527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.112539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173775.120542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.124529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173775.124534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.132544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173775.132556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.136531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.144532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.144546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.148533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.152532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173775.152540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.159550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.163529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.167528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173775.167534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.171527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.175535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.179528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173775.183536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.187528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173775.191540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.195530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173775.199523:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.203530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173775.207531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.211531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173775.215550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.219531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.223528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.227533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173775.231549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.235532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.243539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.243549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173775.252062:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.252076:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173775.259529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.271534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173775.271539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.279541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.279550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.286540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173775.294531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.294548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173775.296528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.296541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173775.304544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.304556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.309553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.309565:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173775.314526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.319543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.327546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.327558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173775.339536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.339544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173775.343540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.347540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173775.351536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.358531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173775.361603:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.366528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173775.370548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.374532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.378543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.387537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173775.396541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.400541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.400548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.405542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173775.405551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.405562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.421542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.421551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.425536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.425548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.429528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.429539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173775.433532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.437533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.441530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.445534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173775.454528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.457530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173775.462534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.462545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.467526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.471533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.475529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.475541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.479527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.484525:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.492534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.497524:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173775.501526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.501534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.505542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.505554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173775.509525:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.514525:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173775.520534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.525530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.527533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.527544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173775.531548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.535542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.543529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.543539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173775.551528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.551537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.560539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.564556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.568548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.568559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173775.576557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.580544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173775.584544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.584554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173775.592542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.601535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173775.609548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173775.609561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173775.613534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.613548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.617527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.617538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173775.621545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.625536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173775.629527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.633540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173775.637541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.641545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.649535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.649545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173775.657529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.657537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173775.661523:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.665532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173775.669548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.673537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173775.677539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.682531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173775.686529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.690554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173775.694549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.698535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.706542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.710536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173775.714533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.719530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173775.724544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.727526:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173775.731543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.731552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173775.743537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.747534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173775.756527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.756540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.760539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.764546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.764556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.764571:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173775.769527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.773545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173775.777545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.781551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173775.788549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.796529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173775.800530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.804534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173775.804539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.808531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173775.812527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.820530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173775.828530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.832540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.832544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.836532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173775.836541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.840534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173775.844534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.848537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173775.852532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.856543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.860530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.865528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173775.867535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.871527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173775.875532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.879531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173775.883537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.891529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.899541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.903524:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173775.903530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173775.907539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173775.907546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.911543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.915543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.915555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173775.923531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.923544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173775.927530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.931530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173775.935528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.940543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173775.944534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173775.944548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.948538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.952542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173775.956541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173775.956547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173775.958535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173775.960539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173775.960545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173775.965529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173775.968535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173775.970536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173775.972530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.976534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173775.980537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173775.981548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173775.981553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173775.984530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173775.988526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173775.992542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173775.993547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173775.996523:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.000532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173776.004533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173776.004543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:0.1:1786173776.004548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173776.008537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173776.012588:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173776.016539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173776.016555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173776.023540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173776.025604:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173776.025610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173776.026535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173776.032532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.036538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173776.039597:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.039607:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173776.040543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173776.044540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.048529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173776.052530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173776.052552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173776.052562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173776.056534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173776.060553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173776.063587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173776.064547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173776.069547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.073577:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:0.1:1786173776.077550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173776.077559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173776.081541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.085544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173776.089538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173776.089688:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173776.096552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173776.096560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173776.102550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:2.1:1786173776.102550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173776.102560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173776.106533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.110537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173776.118555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173776.119549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173776.122551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173776.126540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.130533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173776.130570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173776.130578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173776.134543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.139563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173776.143528:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173776.143535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.147529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173776.151533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.155543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:0.1:1786173776.159546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173776.159552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173776.159554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173776.162600:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173776.167553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173776.171533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173776.171552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173776.176529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.180532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173776.184538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173776.184546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173776.184554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173776.188531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173776.192546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173776.196540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173776.196548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173776.204553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.204565:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173776.208533:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173776.208541:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173776.213541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.213554:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173776.221536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173776.221540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173776.221549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173776.229541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.229550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173776.238540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:0.1:1786173776.238546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173776.246551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.246563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173776.249534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.249553:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173776.254534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173776.254537:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173776.258534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173776.263531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173776.263545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173776.263554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173776.267531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173776.272550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.277554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173776.277588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173776.284539:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.284551:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173776.288557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:0.1:1786173776.288564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173776.292535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.298539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173776.302538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173776.302540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.306535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173776.311534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:0.1:1786173776.314667:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:2.1:1786173776.314667:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:0.1:1786173776.314676:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173776.317556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.322537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173776.327548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:0.1:1786173776.331556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173776.339534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173776.343533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.347545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173776.347548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173776.347552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:0.1:1786173776.347555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173776.351552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173776.356534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.360536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173776.360550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173776.366547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.370548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:0.1:1786173776.373614:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:0.1:1786173776.373621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173776.374535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.378535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173776.382539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.389547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173776.394544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.394552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173776.402547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.406552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173776.410536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.414700:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173776.427526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.435549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173776.435558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.435574:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173776.447547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.447560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173776.453542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.457546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173776.461558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.461569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173776.466550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.470537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173776.478553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.487541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173776.492544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.492552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173776.497547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.503540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173776.512538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.512549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173776.515533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.521536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173776.525537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.528540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173776.532532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.536547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173776.540532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.543566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173776.547548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.552536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173776.560589:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.560599:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173776.568545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.568553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173776.580535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.584533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173776.584539:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.588535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173776.592533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.596545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173776.602545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.608533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173776.612529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.612545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173776.616535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.620535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173776.624541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.628541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173776.635548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.639536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173776.643527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.647532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173776.651533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.656542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173776.660529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.664550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173776.668537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.673537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173776.676531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.680535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173776.684533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.692535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173776.696548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.700533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173776.704539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.708556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173776.712535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.716592:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173776.720530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.724535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173776.732532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.737545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173776.742540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.742553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173776.746533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.746545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173776.750543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.754529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173776.758534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.762529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173776.766532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.770531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173776.774549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.781546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173776.789533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.789543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173776.797547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.797556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173776.802531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.807540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173776.816539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.820545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173776.828534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.828543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173776.838544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.838562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173776.846546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.846560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173776.852553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.856536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173776.860550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.860576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173776.865543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.869550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173776.873544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.877543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173776.882534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.886529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173776.890540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.894548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173776.898546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.902530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173776.910529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.910542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173776.914532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.922543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173776.926546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.930538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173776.930544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.934525:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173776.938538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.942528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173776.946535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.950551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173776.956539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173776.956556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173776.961539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.961548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173776.966532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.971532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173776.978548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173776.982540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173776.986526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173776.986536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173776.990533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173776.994545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173776.998550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.002544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.014530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.018535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173777.018541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.026538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173777.030552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.034550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173777.038550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.042555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173777.046536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173777.055535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173777.059526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.063531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173777.063535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173777.067540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173777.071531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.075534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173777.079533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173777.084531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173777.091549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.095556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173777.100533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.100547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173777.104544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173777.112535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173777.120545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.125554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173777.125563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.129535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173777.133544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173777.133553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173777.138531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.155542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173777.159533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.159549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173777.163535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.163549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173777.171534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173777.179533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173777.183541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.183554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173777.187544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.187555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173777.191533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173777.195551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173777.207548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.215553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173777.215560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173777.219552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173777.223532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.227535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173777.231530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.235533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173777.239549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173777.243549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173777.251540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.251550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173777.260548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.260563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173777.278551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173777.278563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173777.282534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.282542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173777.286537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.286554:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173777.290535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.294545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173777.298537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.302536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173777.310534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.314540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173777.318542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.323532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173777.331530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.331542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173777.335530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.340534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173777.343537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.347547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173777.352547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.356539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173777.365535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.365550:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.370537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.370552:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173777.378538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.382531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173777.386536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.391536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173777.395557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.399553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173777.404529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.412535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173777.424533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.424549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173777.428537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.435539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173777.435546:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.443545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173777.447532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.452533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173777.456543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.456553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173777.460535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.464536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173777.473533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.477534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173777.481529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.481537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173777.486535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.489535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173777.493551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.497549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.502542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.506535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173777.514536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.514544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173777.518539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.522527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173777.526546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.530531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173777.538542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.541672:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173777.546566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.546585:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173777.552542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.561534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173777.563552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.567536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173777.567543:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.571545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173777.575534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.583554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173777.594534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.594552:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.602557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.602572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173777.606536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.610532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.614547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.614558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173777.626550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.630534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.630542:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.638531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173777.642530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.646544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.646553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.650553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173777.660531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.664553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.668575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.668581:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173777.673539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.677538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.690537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.690550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173777.694532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.694546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173777.698543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.702533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173777.706548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.715530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173777.719530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.723535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173777.723545:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.727536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173777.731536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.735538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173777.740569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.744546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173777.749546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.753547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173777.758547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.762535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173777.770555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.770567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173777.775528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.779539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173777.787537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.791537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.795529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.795536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173777.803531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.803545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.807541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.811554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173777.816526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.820535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173777.824545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.829537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173777.837546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.837563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173777.842532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.850533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173777.858544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.858555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173777.862536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.866537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173777.870539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.874533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173777.878550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.882554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173777.887532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.892554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173777.896537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.900532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173777.905545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.908533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.916532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.920532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173777.927535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.927551:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173777.931532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.935535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173777.935545:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.939535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173777.946547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173777.949543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.953531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.953542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173777.957527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.961527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173777.965528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.973534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173777.977552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173777.981527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173777.981532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173777.985534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173777.989532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173777.994528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173777.997530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.005538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.009548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.013536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173778.013543:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.017538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.021544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.025538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173778.029537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.033539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.037529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.043537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173778.046538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.055536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173778.059547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.067530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173778.067534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.067539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.070595:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.075545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173778.075551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.084552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.089542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.089552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173778.093533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.097535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.101530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.105534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.109528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.113544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173778.117533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.121530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.125532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.129532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173778.135534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.137000:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173778.141537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.149538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173778.153531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.157533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173778.165530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.169547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.175534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.175542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173778.183547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.187535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.191534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.191547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173778.199530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.203542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.209547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.209562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173778.214547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.214560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.219549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.227529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173778.235547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.239549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173778.243538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.247534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173778.251531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.260532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.264527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.268529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173778.268534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.273528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.277536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.280529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173778.288553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.288564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.297552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.297562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173778.302549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.306538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173778.310544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.319552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173778.322565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.330555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173778.330563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.334550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173778.338531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.342538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173778.346535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.350537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.358529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.358539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173778.367537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.367554:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173778.371538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.371548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173778.379545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.383576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173778.391532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.395530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.399534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.399543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.403545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.407533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.411528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.417004:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173778.423535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.427550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173778.431531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.435551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173778.447544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.451544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.459541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.459557:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173778.463534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.467551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.471547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.471555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173778.479545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.483539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.488042:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.491535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173778.495532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.499545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173778.503543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.507545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173778.512544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.516542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.521549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.525557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.533546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.533562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173778.538533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.542544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173778.554535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.554547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.570537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173778.570553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.574538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.574548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.578546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.578561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173778.582535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.586543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173778.590533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173778.596595:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.604542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.607543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173778.611528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.611538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173778.619545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173778.623536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173778.627539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.627553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173778.639536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.644539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173778.647544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173778.647558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.655529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.659532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.664547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.664559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173778.667531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.675536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.679532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173778.683556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173778.683563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.687545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.691532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.695545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173778.704551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.708553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173778.712536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173778.716536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173778.720544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.724531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173778.728551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.734530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173778.738558:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173778.738587:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173778.742530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.746537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173778.750531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.754538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.762547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173778.766542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173778.770534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.777530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.781532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.785530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173778.785534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173778.789541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173778.793530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.797545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173778.809538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173778.813534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173778.817569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.817581:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173778.821530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.825532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173778.829527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.833575:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173778.837532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.841530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173778.845531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.849535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173778.854535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.858540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.870537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.874540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173778.874546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.881538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173778.885529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173778.889556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.889563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.893533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173778.897535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.901529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.905533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.909532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173778.913527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.918541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173778.922540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.926543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173778.934541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.938537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173778.942533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.942545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.950533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.955541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173778.959528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173778.959542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.963543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.963552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173778.972529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.972538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173778.982539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173778.982548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173778.986532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173778.986540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173778.994525:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173778.998538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173779.006541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.010531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173779.018546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.018557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173779.022548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.026537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173779.026544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.030537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173779.034532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.038530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173779.042546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.046532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173779.054531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.054538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173779.062543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.062553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173779.067562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.071535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173779.075535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.079550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.088550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.096537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173779.104544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.109531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173779.109538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.113543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173779.117551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.117568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173779.124592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.130533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173779.136575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.136581:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173779.148544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.152554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173779.152563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.156544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173779.160537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.164555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173779.168537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.172530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173779.176526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.180532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173779.184533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.188536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.192534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.196535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173779.207536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.211533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173779.215535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.215552:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173779.219532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.224532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173779.227535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.231549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173779.239549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.243547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173779.248546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.253555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173779.259538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.263539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173779.272548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.277548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173779.277558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.286551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173779.291544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.291564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173779.300535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.304529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173779.308537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.308557:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.317553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.325550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.328575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.333548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173779.333556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.338537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.343532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.351538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173779.355557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.359540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173779.363535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.367537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173779.379533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.383545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173779.383554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.391531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173779.395553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.399532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173779.403542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.407540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173779.412549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.416528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173779.424563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.424581:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173779.436548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.440534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173779.440544:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.440557:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173779.453543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.457537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173779.457548:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.461528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173779.465531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.469541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173779.473529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.477556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173779.482539:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.490552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173779.498543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.502535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.510551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.515545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173779.519536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.519554:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173779.534547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.538536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173779.542537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.542552:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173779.546534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.550546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173779.555542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.559532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173779.568545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.568556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173779.575531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.575544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173779.583542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.587535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173779.599537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.599548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173779.606528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.610534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.616536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.620555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173779.626544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.633544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173779.645556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.645571:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173779.649537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.653533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173779.653538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.658548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173779.666552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.666561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173779.674550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.682550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173779.686554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.690534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.690541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.694530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173779.698534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.706531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173779.710530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.714536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173779.714554:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.718533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173779.723535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.726545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173779.730554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.733660:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173779.738531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.746529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173779.750532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.754535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.759537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.763549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173779.767532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.767541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173779.775530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.775537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173779.779529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.783539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173779.787536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.791531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173779.799549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.807551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173779.811529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.811539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173779.815530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.815542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173779.819548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.822612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173779.827539:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.844541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.848538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.848551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173779.853547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.853555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173779.858528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.862547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.866527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.866538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173779.876541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173779.876555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.879535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.883530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173779.887535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.891543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173779.895529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.903541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173779.911529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.911542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173779.915534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.919530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.923535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.928548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173779.933535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.933545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173779.938533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.943541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173779.950545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.954550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.963540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.967544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173779.971549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173779.971561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173779.975533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173779.975541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173779.980528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173779.984533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173779.988531:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173779.988533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173779.992527:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173779.993543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173779.996534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173779.996549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173779.996557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173780.000529:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173780.000533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173780.009535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.013546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.013553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173780.013559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173780.013561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.017543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173780.017551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173780.017555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173780.018994:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.019009:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.021553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.021559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173780.027557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173780.027569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.029532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173780.033538:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173780.033541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.033545:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173780.037542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173780.037545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.041542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.041548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173780.049548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.057540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.057551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173780.061529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.061532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.061536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173780.061537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.061538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173780.069531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.069542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:2.1:1786173780.069550:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.069551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173780.069559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173780.073532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.073542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.073550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173780.077527:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173780.077532:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:2.1:1786173780.077533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.077535:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173780.077540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.081535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.081545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173780.085532:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173780.085538:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173780.086536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.089549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173780.089553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173780.093553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.097535:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.097544:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173780.100590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173780.101527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173780.105534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.105542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.105548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173780.112550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.113540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173780.116536:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173780.116543:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173780.120532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.120544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173780.120545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.124524:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.124527:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173780.124548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.128534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173780.128550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.136545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.136553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173780.140538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173780.140551:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173780.143590:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.144552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173780.144561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.144566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173780.148531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.148555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173780.153544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173780.153547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.153550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173780.158550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173780.158553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.163548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173780.163548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.163558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173780.172544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173780.172558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173780.177555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173780.177555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.177564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173780.177568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.177573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173780.186539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173780.186544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173780.186557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.191530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173780.191534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.191548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173780.191556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.191562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173780.195535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173780.199532:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173780.203540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173780.203547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.203548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173780.203561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.207532:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173780.207539:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173780.207543:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173780.211544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173780.211552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.215532:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173780.219544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.219550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173780.219554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.219566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173780.223534:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173780.223542:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.223548:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173780.227549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173780.227558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.231540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173780.231547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173780.231554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173780.235529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.239528:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173780.239534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173780.243528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.243530:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.243537:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173780.247534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173780.247541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173780.251545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.251551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173780.251553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.255543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173780.255544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173780.260534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.260547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173780.260553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173780.268544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.269545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173780.273539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.273541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173780.273548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173780.273553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173780.273555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173780.282548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173780.282552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.282556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173780.286532:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173780.286533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.290534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173780.290537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.290540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173780.290548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.294534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173780.294538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.294541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173780.298538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173780.298547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.302529:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173780.306532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.306538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173780.306549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.306560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.310526:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173780.310532:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.310536:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173780.314532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173780.314541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.318544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173780.318551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.318554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173780.322536:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173780.322543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173780.322557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173780.327531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.327543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.327550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173780.333570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.336536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173780.340537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.340538:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.340546:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173780.344536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.345535:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173780.348534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.348539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173780.348547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.352543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173780.352544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173780.357537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.357546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.357555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173780.361539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.361541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173780.366535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173780.366540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173780.366544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.370544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173780.370547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173780.375542:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.375552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.375561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173780.384545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.384585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173780.388566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.388578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173780.392546:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.392571:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.396558:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173780.400545:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.401549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.401555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173780.404548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.404551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173780.412549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.412556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173780.417556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.421549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173780.421559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173780.421560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.421573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173780.425533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.425549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173780.429553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173780.433539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173780.433547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173780.433550:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173780.441551:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173780.445547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173780.448662:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.448670:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173780.448674:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173780.448950:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.448968:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173780.457536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.457567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173780.457580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173780.461553:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.465548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173780.470540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173780.470543:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.470550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173780.474546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173780.482535:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173780.486530:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173780.486536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.486537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173780.490529:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173780.490536:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:2.1:1786173780.490536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.490541:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173780.490543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173780.494535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.494537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173780.498547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.498556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173780.503547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173780.503558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.511543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173780.515534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.515546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.515554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173780.519533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173780.519541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.519550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173780.519558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.519566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173780.522663:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173780.523531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173780.531544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.532537:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.532542:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173780.535555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173780.539531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.539558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173780.539569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.539577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173780.547535:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173780.548538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.552541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173780.552547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173780.552550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173780.561532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.561533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173780.561541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.561542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.561549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173780.565532:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173780.565535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173780.569528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.569533:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:2.1:1786173780.569538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.569542:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173780.569549:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173780.573532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.573537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173780.573545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.577548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173780.577553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173780.582547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.582551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.582559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173780.586536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.586540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173780.590533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.590538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173780.595533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.595543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.598543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173780.603540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173780.603544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.603547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173780.603557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.607522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173780.614531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.614545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.614553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173780.618529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.618551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173780.622537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.622545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:2.1:1786173780.622549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.622555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173780.622562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173780.626541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173780.626550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173780.626550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173780.633539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.633542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173780.638562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173780.638578:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173780.638755:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.638761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173780.638763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173780.644083:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.647529:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.647535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173780.647536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.651535:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173780.659532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.659540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173780.659547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173780.663549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.663586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173780.667548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.667551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.667559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173780.667564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.667565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173780.672543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.672551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173780.672552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.676536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173780.676538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.680548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.680557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173780.684546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.684556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.688546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173780.692533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173780.692538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.692539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173780.700528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.700534:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173780.704530:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173780.704532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.704538:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173780.708530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173780.708535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173780.708535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.708545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173780.708550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173780.712532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.712535:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173780.716524:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173780.716530:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173780.716550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.720533:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173780.720548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173780.724532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.727620:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.727626:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173780.727630:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173780.728542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173780.732551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173780.732559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173780.736556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.740530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.740543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173780.744532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.744540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173780.744546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173780.748537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.748544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173780.752543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.756543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:2.1:1786173780.756544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.756550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173780.756555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173780.765544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173780.765548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.765550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173780.769533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.769540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173780.777541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.777549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:2.1:1786173780.777555:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.777559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173780.781541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173780.785532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.785544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.786548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.786558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173780.786564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173780.789534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.789536:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.789544:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173780.793543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173780.793544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173780.797538:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.797544:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173780.802549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173780.805537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173780.805555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173780.809536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.809549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173780.809556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173780.813524:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173780.813543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173780.817532:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173780.817538:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173780.817538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.825528:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173780.825531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173780.829535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.829548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173780.829555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173780.833525:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173780.833529:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.833532:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173780.833536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173780.833542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.841533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173780.841540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.845532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.845536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173780.845541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173780.849537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173780.849541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.850538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173780.850542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173780.850545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173780.853537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173780.853541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.861523:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173780.861527:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173780.861529:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173780.861547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173780.869541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.877527:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.877532:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173780.877537:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173780.877570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.877580:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.881545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173780.881546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173780.881553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173780.881558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173780.881560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173780.885531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173780.885538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.885544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173780.885551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173780.885556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173780.889527:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.889532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173780.889543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173780.889552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.893539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173780.899549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173780.899555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.899559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173780.908534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173780.909536:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173780.912538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173780.912548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:2.1:1786173780.912551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.912558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173780.920541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173780.920547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173780.920556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:2.1:1786173780.920558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.920561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173780.920568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173780.925536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173780.925545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.925546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173780.925554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173780.925558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173780.929539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173780.933528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173780.933535:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173780.933543:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173780.937546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173780.937555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.937561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173780.945537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173780.945553:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.946550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173780.949548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.953528:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:2.1:1786173780.953532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.953535:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173780.953543:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173780.953547:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173780.953551:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173780.961537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173780.961546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.965537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173780.965544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173780.965548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173780.969540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173780.969545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173780.969559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173780.974529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.974533:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173780.974541:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173780.974541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.978541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173780.986541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173780.986547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173780.986556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173780.986557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173780.986571:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173780.990536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173780.994531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173780.994541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173780.994550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173780.994554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173780.998539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173780.998544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:2.1:1786173780.998551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173780.998552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173781.002531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173781.006545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173781.006552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173781.006552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.010533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.010538:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173781.015547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.015557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173781.015557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.017533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173781.017544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.021546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173781.021548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.021553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173781.026547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173781.026548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.031541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173781.031544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.031548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173781.035530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173781.036554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173781.039540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.039546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173781.042657:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173781.043530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173781.043540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.046580:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173781.046584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173781.047535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173781.054535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173781.058539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.058547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173781.062529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.062538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.062548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173781.062555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173781.070551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.070594:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.070598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173781.074531:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173781.074537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173781.078536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173781.078548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173781.078552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.078560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173781.078566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173781.082535:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.082543:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173781.083534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.085999:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.086551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173781.090547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.090555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173781.090563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173781.094548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.094549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173781.099528:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.099535:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173781.099553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173781.103529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.103533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173781.109534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.109540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173781.112527:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173781.112541:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173781.112556:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173781.116529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.116535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.116542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173781.120531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.120535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173781.124534:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173781.124534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.124543:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173781.128529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.128550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173781.132536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.132541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.132548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173781.140539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173781.140544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173781.149544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173781.149546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.149550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173781.149556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173781.149568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.154543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173781.154543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.154555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.154558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173781.154566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173781.159534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.159560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.159569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173781.162904:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.167527:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173781.171530:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173781.171531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.171540:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173781.175527:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173781.175529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173781.179532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.179535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.179539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173781.183535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173781.183540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173781.187528:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.187532:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173781.187547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.195530:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173781.196570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173781.199530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.199541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173781.199549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173781.203538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173781.203544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:2.1:1786173781.203546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.203549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173781.203558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.207527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.207535:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173781.215531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.215536:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.215545:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173781.219528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.219529:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173781.223533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.223536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.223544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173781.223547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.223549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173781.231551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173781.231553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.231559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173781.235536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.235548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173781.239547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.239554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173781.239554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173781.239566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.243541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173781.247532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.247538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.247546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173781.252534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.252543:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173781.256535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.264545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.264546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.264555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173781.264558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173781.269543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.269545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.269553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173781.269555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.269559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173781.274534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.274535:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.274542:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173781.278544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.282540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173781.282548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.282552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173781.286548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173781.286557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173781.290539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173781.298544:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.298554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.298562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173781.306536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173781.306539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173781.310538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.310546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:2.1:1786173781.310549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.310557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173781.310561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173781.314544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173781.314545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.314549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173781.314555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173781.314559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173781.318545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.322543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:2.1:1786173781.322544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.322551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173781.322558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173781.327537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.327543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173781.327550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173781.331541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.335541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173781.339533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173781.339537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.339539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173781.343532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173781.343542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173781.347527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.347533:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.347539:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173781.355538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173781.355540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173781.359548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.359554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173781.360545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.363527:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173781.363543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173781.367540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.367550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173781.367551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.367558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173781.371526:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173781.376543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173781.376549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173781.376559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173781.381553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.382534:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173781.385538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.385551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:2.1:1786173781.385557:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.385559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173781.389528:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173781.389534:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.389539:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173781.393537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.393545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.397551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173781.397558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.397560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173781.400549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.404541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173781.404547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.410549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.410555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173781.410566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.413612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173781.414551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.424541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.424547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173781.435703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173781.441535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.441542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173781.441543:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.441553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.441557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173781.445544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173781.445545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.445554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173781.445555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.445562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173781.445572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173781.449528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.449554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173781.449564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.449570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173781.453534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.453543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173781.456599:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.457554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.457563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173781.460535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.460547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173781.468532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.468539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173781.472536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173781.472554:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173781.477538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.477544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173781.477551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.477556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173781.477560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173781.481542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.481551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173781.481554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173781.487591:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173781.496533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.496545:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173781.504535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.504539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173781.504549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.508556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173781.508558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173781.512535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.512545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.512552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173781.516539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173781.516552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173781.520534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.520535:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173781.520544:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173781.526556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173781.526597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.534542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.534548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173781.535532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.543534:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173781.543535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173781.547534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173781.547541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.547546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.547554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173781.547557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173781.551526:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173781.551528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.551531:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173781.551537:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173781.551538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173781.555537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173781.555544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.556527:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.556532:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173781.560119:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173781.560127:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.560130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173781.565541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.565546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173781.565552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173781.571536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.571537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.571543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173781.571547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.571548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173781.576535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.576541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173781.576549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173781.581535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173781.581545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.585536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.585544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173781.589551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.594531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173781.594545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173781.598533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.598536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.598539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173781.606540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.607530:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173781.610539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173781.610546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.610548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173781.610558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173781.615544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.615559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173781.615560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.615570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.615577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173781.615582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173781.620535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.620552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.620559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173781.624556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173781.624557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173781.629532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.629545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.629552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173781.633549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173781.633555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.638533:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173781.638535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.638538:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173781.642539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.646533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173781.646539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.646544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173781.654536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173781.659542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173781.659556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173781.663558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.663576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173781.664021:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.664028:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173781.664032:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173781.668037:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.668048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173781.668588:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173781.677555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.681538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173781.685536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.685537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173781.685557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173781.689530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.689535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173781.693531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.693548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.693555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173781.697532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.697553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173781.701541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173781.705531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173781.705537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.705544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173781.705549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173781.708734:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.708742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173781.709667:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.718537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173781.718548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173781.726530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.726536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:2.1:1786173781.726541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.726543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173781.726546:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.726549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173781.730530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.730534:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.730542:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173781.730549:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173781.734539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.734549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.734556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173781.738529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.738539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173781.742541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.742548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173781.746528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.746531:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173781.746539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.750546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173781.750548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.750552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173781.754538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173781.754541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173781.758537:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.758544:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173781.762532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173781.762552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173781.762563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.766534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173781.766541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173781.770536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.778534:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173781.778537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.782529:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173781.782533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.782534:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173781.786526:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173781.786530:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.786533:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173781.786549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173781.786557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.790528:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173781.790551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173781.794547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.798543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.798546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.798555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173781.798558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173781.806555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.806561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173781.806603:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173781.810550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173781.811545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173781.814530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.814537:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.814550:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173781.818535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.818544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173781.823551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.823567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173781.823576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173781.828536:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173781.828542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.832542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.832548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173781.832554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.836546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173781.836562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173781.841527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.841529:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.841535:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173781.845536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.849528:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173781.849533:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173781.849537:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173781.853535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173781.853545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173781.857530:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173781.857536:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.857541:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173781.861539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.865542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173781.865552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173781.871551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.871554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173781.871562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173781.879531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.879537:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173781.883533:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173781.883536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.883541:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173781.887545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173781.887547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.887553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173781.887555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.887559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173781.891542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.891552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173781.896524:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173781.896528:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173781.896545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.900527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.900529:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173781.904528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173781.913533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173781.913542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.913550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173781.913557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173781.917533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173781.921530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173781.921535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.921542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173781.921550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173781.921556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173781.921563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.921569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173781.924546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173781.924550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173781.929555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.933530:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173781.933537:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173781.941532:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173781.941538:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:2.1:1786173781.941539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.941542:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173781.945537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173781.949535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173781.949540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173781.949544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173781.949545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173781.949553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.953551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173781.953552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.959542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173781.959545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.959549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173781.962620:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173781.963551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173781.967549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173781.971538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173781.971543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173781.975534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.975553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173781.979534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173781.979543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173781.979551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173781.983541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173781.983549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173781.991540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173781.998537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173781.999549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173781.999556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173781.999559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173782.002525:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173782.002532:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173782.002532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.006535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173782.006543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:2.1:1786173782.006545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173782.006549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173782.006551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.010528:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173782.010532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173782.014532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.014540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.014547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173782.018531:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173782.018538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173782.022524:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173782.022528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173782.022546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.026541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173782.026546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.030531:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173782.030538:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173782.034551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173782.034560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.038532:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173782.038536:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.038538:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173782.046548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173782.050532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.050663:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173782.054535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173782.054551:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.055548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.055554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173782.055558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173782.059529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.059548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.059556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173782.065669:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173782.066553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173782.070536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173782.074535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.074551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.074561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173782.074567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173782.078548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.082544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:2.1:1786173782.082547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173782.082551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173782.086538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173782.090536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.090541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173782.090553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173782.094553:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.094563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173782.102556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173782.102564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173782.103547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.106544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173782.107532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.109576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.109582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173782.110537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173782.110560:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.114534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173782.118533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173782.126544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.126546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.126555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173782.130536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173782.130543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173782.130548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173782.134552:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.134558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173782.134570:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173782.138534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.138538:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173782.138544:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173782.142533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.142537:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173782.146539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.146545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173782.146547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173782.151550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173782.151558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173782.156529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.156542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.156551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173782.160535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173782.161532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173782.164532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173782.168548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:2.1:1786173782.168549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.168557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173782.168563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173782.173534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.173546:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.173553:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173782.177544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173782.181545:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173782.185537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173782.185544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.185554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173782.189530:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173782.189531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173782.193545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.193555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173782.193563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173782.197539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173782.197540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173782.205540:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.205550:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173782.209552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173782.213531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173782.213537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173782.213544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173782.213553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173782.213564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173782.217536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173782.225547:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173782.225556:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.225560:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173782.229543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.233529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173782.233535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:2.1:1786173782.233538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.233539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173782.237549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173782.241540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.242542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173782.245550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173782.249539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.249549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173782.249557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173782.253539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173782.257527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.257536:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.257544:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173782.261534:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173782.261536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.265532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.265542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173782.266538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173782.269536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.269543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173782.273545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.273550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.273557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173782.281551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173782.281554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173782.286529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173782.286538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173782.286548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.286558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173782.286566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173782.290547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.294542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:2.1:1786173782.294544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.294550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173782.294555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173782.298539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173782.298545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173782.302542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.302552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173782.302560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173782.307545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173782.307548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173782.307551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173782.310587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173782.311542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173782.314581:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173782.314587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173782.319540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173782.323535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173782.323545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173782.327535:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173782.327544:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173782.328532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173782.331542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.331550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173782.335533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173782.335547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173782.335558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173782.343545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.347547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173782.351533:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.351540:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173782.351543:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173782.351555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173782.351573:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173782.355555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173782.359538:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.359545:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173782.363537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.363542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173782.367536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.368565:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.368575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173782.373558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173782.374544:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173782.377683:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.378565:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.378576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173782.382546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.382555:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173782.392557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.393542:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173782.393550:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173782.396556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.404527:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173782.404533:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.404537:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173782.408554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173782.408567:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173782.416545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.416548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173782.416559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.416566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173782.420551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173782.421617:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173782.425552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173782.425556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.425568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173782.425575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173782.425577:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173782.425580:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.425588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173782.430543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173782.435544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173782.435559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173782.439544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173782.439551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173782.447540:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173782.447543:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173782.447581:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.452541:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173782.452550:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173782.452553:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.452556:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173782.452576:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173782.457556:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.466542:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.466548:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173782.470560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173782.470566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173782.474532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.474535:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.474544:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173782.478532:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173782.479544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.482533:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.482541:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173782.482553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.486553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173782.487552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.491563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173782.491565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.491576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173782.495582:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.499545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173782.503544:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173782.503546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.503556:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173782.507545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.512203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173782.515548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173782.515553:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.515557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173782.520552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173782.524537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.524539:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.524548:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173782.534546:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173782.537545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.542536:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173782.542544:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.542546:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.542567:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173782.542570:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.547544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173782.550547:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.550701:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.550712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173782.554534:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173782.554546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173782.562533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.562536:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.562545:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173782.565584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173782.566533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173782.570541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.570544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.570552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173782.574546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.582537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173782.582544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173782.586546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.586547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173782.586556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173782.590546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173782.590549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.590555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.590561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173782.590564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.590567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173782.595542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.595545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.595553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173782.600536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173782.600544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.604541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.604554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173782.612551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173782.612569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173782.620546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173782.624546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.624553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173782.624557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173782.625535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173782.625547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.628534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173782.628536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.628547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173782.628556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173782.628562:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173782.632552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173782.632555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173782.632561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173782.640535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173782.640538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173782.644546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.648539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173782.648545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173782.648548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173782.648565:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173782.652539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.652544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173782.652558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173782.656545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.656551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173782.660528:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173782.660533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.660536:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173782.664527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173782.664539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173782.668529:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173782.668532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.668534:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173782.672533:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173782.672536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.676530:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173782.676533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.676536:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173782.680528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.680533:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173782.684545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.684551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173782.684551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173782.688534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.688540:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173782.691916:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.692534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.692542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173782.696546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173782.696549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173782.708540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173782.708541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173782.708548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173782.712530:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173782.712533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.712536:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173782.712542:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.712543:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173782.716530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.716545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173782.724560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.725547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173782.725554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173782.728548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.728550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173782.732828:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.732842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173782.733054:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.737630:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173782.738602:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.746531:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.746540:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173782.746554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173782.750542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173782.752034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173782.755561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.759014:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.759020:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173782.763816:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173782.764008:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173782.770537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.774557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173782.774565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173782.778567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173782.778568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173782.786557:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.786559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.786570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173782.791571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173782.792557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173782.800567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.800568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173782.800589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173782.805562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173782.805573:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173782.805579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173782.805584:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173782.810542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173782.814549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.814565:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173782.819543:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.823541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173782.823548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173782.827529:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173782.827551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.835545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.835552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173782.839592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173782.844555:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.844710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173782.844720:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.844727:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173782.853539:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173782.853557:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.857534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173782.861550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.861553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.861562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173782.861568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173782.861571:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173782.866539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.866550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.866558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173782.870539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173782.870549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.875534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173782.875540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173782.875541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.879543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173782.879544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173782.883701:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173782.883715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173782.883737:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173782.888551:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.888559:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173782.893539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173782.893544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.893545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173782.898542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173782.898545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.907543:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.907550:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173782.907558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.916550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173782.916556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.916560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173782.916565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173782.920566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.924542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173782.924548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173782.928538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.929534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.929541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173782.932542:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173782.933533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.936537:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173782.936545:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173782.940548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173782.940559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173782.944545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173782.944551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173782.944554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173782.948560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.952549:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173782.952558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.957544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173782.957548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173782.957551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173782.962535:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173782.962549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.967555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173782.967562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173782.968536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173782.972544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173782.972550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173782.981545:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173782.981566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173782.981576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173782.986550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173782.986555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.986559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173782.986564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173782.986573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173782.992533:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173782.992536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173782.997534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173782.997549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173782.998539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173782.998544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173783.007536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173783.010538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.010542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173783.010551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173783.014536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173783.014537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.014545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173783.014546:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.014550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173783.018537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173783.018549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173783.022535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.026536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.026541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173783.026545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.032530:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173783.032537:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.032542:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173783.036551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.040542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173783.040546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.040555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.040560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173783.044532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.048536:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173783.049534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173783.052537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.052547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173783.052548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.056543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173783.056544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173783.061539:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.061544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173783.061556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173783.065562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173783.066542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.070552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173783.070564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.070565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173783.076541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173783.076553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173783.081535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.081543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.081551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173783.085538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173783.089534:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173783.093562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173783.093569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173783.094546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.097543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173783.097558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173783.102543:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.106549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173783.106556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173783.110538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173783.110547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.114539:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173783.114546:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173783.118536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173783.118559:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.122554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173783.126547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.126555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173783.126557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173783.130536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.134548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173783.138532:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173783.138533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.138540:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173783.142538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173783.142547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.146550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.146558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173783.150534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.153617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173783.154544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173783.158537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.158545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.158555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173783.166544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.170528:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173783.174532:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.174541:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173783.174548:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173783.175538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173783.175547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173783.178527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.178534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.178541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173783.182533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173783.182538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173783.186546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.190541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:2.1:1786173783.190542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.190549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173783.190553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173783.195539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173783.195544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173783.196527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173783.199540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173783.203527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.203532:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173783.207536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.207541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173783.211534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.211541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173783.211548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173783.211554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173783.219537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.226535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173783.226541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173783.230534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173783.230538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:2.1:1786173783.230545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.230547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173783.230555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173783.234529:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173783.234534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.234536:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173783.234545:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173783.234548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.238535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.238541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173783.238560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173783.247534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.247540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173783.256535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173783.256546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:2.1:1786173783.256547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.256554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173783.256561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173783.260542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.260549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:2.1:1786173783.260552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.260557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173783.260563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173783.264545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173783.264550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173783.272542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173783.272563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173783.276535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173783.280533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173783.280535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.280542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173783.280546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.280547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173783.284533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.284535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173783.284542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173783.287588:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.288546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173783.292549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.296525:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.296529:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173783.296532:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173783.296547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173783.300538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.300543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173783.304544:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173783.304550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173783.304569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173783.309542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173783.309545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173783.309553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173783.313532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173783.313538:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173783.317539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.317546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173783.317556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173783.321525:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173783.321532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.325530:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173783.325537:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173783.326540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173783.329531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.329542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173783.333531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173783.333546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.333552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173783.337537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173783.337542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173783.341533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.341538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173783.341545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173783.351530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.355528:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173783.359533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173783.359540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173783.359548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173783.363530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.363534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173783.367542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.371551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:2.1:1786173783.371554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.371558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173783.371562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173783.375529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173783.383533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.383545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.383555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173783.387539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173783.391530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.391537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.391548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173783.391548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173783.391553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173783.395529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173783.395544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.395553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173783.399545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173783.399546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173783.404577:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.409545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173783.409553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173783.411538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.416567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173783.419589:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.419594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173783.420535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173783.420546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173783.423900:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.424534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173783.424543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.424549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173783.428541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.432541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173783.436532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.436534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.436542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173783.440527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173783.440532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173783.444125:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173783.444545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.444551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173783.448550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.449537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173783.453534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173783.453540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173783.453541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.458529:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173783.458533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173783.462530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173783.466541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:2.1:1786173783.466541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.466548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173783.466553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173783.470536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.471527:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173783.471531:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173783.474538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173783.474557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173783.478533:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.478541:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173783.481965:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173783.481978:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.491566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173783.491575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173783.491578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173783.496532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.499531:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173783.499535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173783.499537:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.499544:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173783.499544:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173783.499559:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173783.507534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173783.507540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173783.507551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173783.507556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173783.511535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.511536:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173783.515536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173783.515549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173783.516530:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.516534:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173783.516537:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173783.519530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.519533:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.519539:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173783.523534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.524543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173783.531543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173783.531548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173783.531555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.535536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173783.535569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173783.539549:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173783.539552:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.539561:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173783.539570:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.543561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173783.548540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.548545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173783.548558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173783.552570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.556541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173783.560532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.560544:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.560556:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173783.564528:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173783.564546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173783.568544:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173783.568546:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.568561:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173783.572558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.577530:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173783.577539:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173783.577543:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173783.581531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.585538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173783.585557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173783.589563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.593535:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.593544:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173783.597539:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173783.597547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.604540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173783.604547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173783.605546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.608669:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173783.609595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173783.613538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.613540:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.613552:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173783.617549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173783.617597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.621538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173783.621545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173783.621546:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173783.626544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173783.630554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173783.634544:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.634546:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.634559:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173783.638540:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173783.639589:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.642529:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173783.642531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.642538:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173783.646530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.646538:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173783.650537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.650545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.650552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173783.655545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173783.655546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173783.660547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.660554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173783.662532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173783.667577:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.668546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173783.672547:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173783.680531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.681548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173783.681556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173783.681563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173783.681566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173783.681568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173783.684532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.688536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173783.688542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.688545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173783.688553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173783.688561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.692533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173783.692548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.696528:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.696532:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173783.700547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173783.700555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.704527:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173783.704532:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.704535:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173783.713547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173783.713555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173783.717546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.721544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.721551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173783.721554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173783.721560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173783.721574:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173783.725537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.726532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.726538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173783.733550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.741527:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173783.749530:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173783.749538:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173783.749542:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173783.750552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173783.750564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173783.753536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.753543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.753550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173783.753551:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173783.753555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173783.757159:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173783.757169:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173783.757173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173783.757549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173783.757563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.762526:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.762531:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173783.762547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173783.766541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173783.770532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173783.770537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.770539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173783.775537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173783.775548:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.779533:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173783.783543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173783.783545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.783551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173783.783559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.787530:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173783.787537:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.787543:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173783.791529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173783.791538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173783.795530:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173783.795539:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173783.795546:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173783.799530:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173783.799532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173783.799542:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173783.803528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.803543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173783.803552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173783.807527:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.808531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173783.811526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.815529:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:2.1:1786173783.815532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173783.815534:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173783.815541:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173783.818594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.819528:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.819536:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173783.823528:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173783.823532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173783.827530:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173783.827535:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173783.831535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173783.831547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.832523:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173783.835531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.835534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173783.835541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173783.838536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173783.838549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173783.842531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.843537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.843541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173783.846551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.854546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173783.854555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.854562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173783.858558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173783.858572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173783.862531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.862549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173783.862559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.862567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173783.866536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173783.866543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173783.871538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.871547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.871556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173783.880531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.880537:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173783.884527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173783.884536:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173783.884542:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173783.888531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173783.888536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.888542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173783.888550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173783.888555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173783.892530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.892542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173783.896531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173783.900528:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.900543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173783.900551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173783.900558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173783.904538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173783.904544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.904551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173783.908539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.908543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173783.913529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.913538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.913546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173783.917528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173783.917544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173783.921535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173783.925541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.925543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.925550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173783.925554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173783.933639:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173783.933660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173783.934532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173783.938549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173783.938569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.941609:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173783.941613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173783.942547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173783.942557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173783.946534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173783.946535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173783.954536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.954547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173783.954556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173783.960544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173783.960551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173783.960554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173783.964544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173783.964551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173783.964562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173783.968526:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173783.968533:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173783.968537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173783.972162:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.972538:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173783.976537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173783.976540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173783.976544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173783.982534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173783.982549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00000080:00000010:2.1:1786173783.990550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173783.991529:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173783.991536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173783.995542:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173783.995549:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173783.995553:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173783.995557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00000080:00000010:2.1:1786173783.995567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.002547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173784.002550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173784.006566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173784.006605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173784.010529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173784.010549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173784.020540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173784.030550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:3.1:1786173784.030555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:3.1:1786173784.030564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173784.035535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173784.035542:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:3.1:1786173784.035546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173784.035552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173784.035561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173784.035567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173784.039544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.039548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173784.039555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173784.044536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173784.044537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:2.1:1786173784.044550:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:3.1:1786173784.049533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:3.1:1786173784.049538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173784.054533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173784.054540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173784.054552:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.059536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173784.059541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173784.059550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173784.068530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:3.1:1786173784.068530:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173784.072536:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.072543:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173784.082526:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:3.1:1786173784.082533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:3.1:1786173784.082540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173784.082543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173784.082558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173784.082562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.086526:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173784.086538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:3.1:1786173784.086538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173784.088530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.088537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:2.1:1786173784.088539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173784.088543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173784.088545:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173784.092535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173784.092540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173784.092542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173784.092550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:3.1:1786173784.101533:0:2394:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:3.1:1786173784.101538:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:1.1:1786173784.105561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173784.106534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173784.106546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173784.115541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:3.1:1786173784.115548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173784.115549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173784.115554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173784.115564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173784.119536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.119545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173784.119553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173784.119560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173784.123539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:3.1:1786173784.123572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:3.1:1786173784.123575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173784.128533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.128538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173784.128543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:3.1:1786173784.134539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173784.134545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:3.1:1786173784.134546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173784.134550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173784.134559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173784.138530:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173784.138539:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173784.142533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173784.142542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:3.1:1786173784.147533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173784.147546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173784.147554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173784.151544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173784.151552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173784.151559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173784.160534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:3.1:1786173784.165536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:3.1:1786173784.165542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173784.165561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173784.174536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.174543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:3.1:1786173784.174543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173784.174546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173784.174555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.174561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:3.1:1786173784.178530:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:3.1:1786173784.178535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173784.178543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173784.178545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:1.1:1786173784.178552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173784.182553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173784.182555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173784.182559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173784.182566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173784.191540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:3.1:1786173784.196529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173784.196540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173784.196542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.196550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173784.196556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173784.206540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173784.206548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173784.206553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173784.207536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173784.207546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00000080:00000010:2.1:1786173784.210556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173784.211534:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173784.211540:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173784.211545:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:3.1:1786173784.211568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:3.1:1786173784.211576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173784.215536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:2.1:1786173784.215540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173784.215543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00000080:00000010:2.1:1786173784.215548:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:3.1:1786173784.215549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173784.215563:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173784.221540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.221550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173784.221552:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:3.1:1786173784.229541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:3.1:1786173784.229550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173784.233537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173784.237536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.237541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173784.237542:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.237551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173784.237558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173784.242531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:3.1:1786173784.242538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173784.246531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.246548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173784.246558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173784.250537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.250543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:3.1:1786173784.257546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173784.257547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:3.1:1786173784.257555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173784.265539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173784.265690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173784.267552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.267561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173784.268541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173784.270552:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173784.270561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173784.270567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:3.1:1786173784.270570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173784.270575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173784.274537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173784.279544:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:3.1:1786173784.283576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:3.1:1786173784.283586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173784.283644:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173784.286608:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173784.287550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173784.292540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173784.292548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173784.300551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:3.1:1786173784.300576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173784.308546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173784.312536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173784.312547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173784.312550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173784.312560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173784.312567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:3.1:1786173784.312612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:3.1:1786173784.312623:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173784.316537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173784.316552:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173784.316554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173784.320542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00000080:00000010:2.1:1786173784.320549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173784.320551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173784.325539:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:3.1:1786173784.325549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173784.330535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173784.330546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:2.1:1786173784.330549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.330557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173784.335547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173784.339533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:3.1:1786173784.339543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:3.1:1786173784.339554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173784.342611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173784.343529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173784.347532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173784.347542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173784.347552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173784.351537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:3.1:1786173784.351540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173784.354533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.359541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:2.1:1786173784.359542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173784.359546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173784.359549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:3.1:1786173784.363542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:3.1:1786173784.363547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173784.366581:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173784.366588:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173784.367539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173784.371534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:3.1:1786173784.374537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173784.374547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173784.374554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173784.377685:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173784.377846:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173784.377852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173784.387538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173784.387553:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.391545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173784.394532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:3.1:1786173784.394535:0:103738:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173784.394543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173784.394546:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:3.1:1786173784.394550:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173784.402537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.402546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173784.402554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:3.1:1786173784.406531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173784.406546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173784.410543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.410545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173784.410552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173784.410558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173784.410558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:3.1:1786173784.414534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:3.1:1786173784.414540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173784.418535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173784.418547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173784.418558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173784.423533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.423557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173784.423567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173784.428550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:3.1:1786173784.432546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173784.436541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.436545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173784.441546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.445529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173784.445544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:3.1:1786173784.445584:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:3.1:1786173784.445593:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:2.1:1786173784.449529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.454534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:2.1:1786173784.454543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00000080:00000010:2.1:1786173784.458547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:3.1:1786173784.458611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173784.458795:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173784.458801:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173784.462530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173784.467565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.467727:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173784.467733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:3.1:1786173784.472537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:3.1:1786173784.472543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173784.472552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173784.473538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00000080:00000010:2.1:1786173784.481538:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173784.481554:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:3.1:1786173784.485540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173784.485542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173784.486537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173784.489537:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173784.489544:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173784.489546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173784.494544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173784.494550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173784.495541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.498533:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:2.1:1786173784.498547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:3.1:1786173784.498548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:3.1:1786173784.498558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173784.502527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173784.502538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173784.506568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173784.510544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173784.510552:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173784.510553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:3.1:1786173784.514541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173784.514610:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.523558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173784.523568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173784.528534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.531536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:2.1:1786173784.531543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:3.1:1786173784.531545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173784.531551:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:3.1:1786173784.531552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173784.535553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173784.535556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173784.540543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00000080:00000010:2.1:1786173784.540548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173784.540550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:3.1:1786173784.545542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173784.546623:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173784.555530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173784.555540:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173784.555546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173784.555556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173784.559557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173784.563540:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:3.1:1786173784.563543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173784.563555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:2.1:1786173784.563570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:3.1:1786173784.563582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173784.571534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.572526:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173784.572533:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173784.580553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:3.1:1786173784.580569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:1.1:1786173784.584534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:1.1:1786173784.584540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173784.585535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173784.588535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173784.588536:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:3.1:1786173784.588550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:3.1:1786173784.588560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00000080:00000010:2.1:1786173784.592545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.592548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173784.592558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173784.597850:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173784.597859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173784.601532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:3.1:1786173784.601542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173784.603556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00000080:00000010:2.1:1786173784.607526:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.607547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173784.607558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173784.612529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:3.1:1786173784.621539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173784.621542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:3.1:1786173784.621546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:1.1:1786173784.630546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173784.631533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173784.631547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:3.1:1786173784.634537:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:1.1:1786173784.634541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173784.634548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173784.635547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173784.635560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173784.638528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173784.638534:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:2.1:1786173784.638541:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173784.638542:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173784.638548:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173784.642530:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:3.1:1786173784.642539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:3.1:1786173784.642548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173784.645538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173784.645610:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173784.649526:0:92301:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00000080:00000010:2.1:1786173784.649528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.649532:0:92301:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:3.1:1786173784.653533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:2.1:1786173784.653542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173784.657543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173784.657549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173784.661550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173784.661559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173784.665532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:3.1:1786173784.665538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:3.1:1786173784.665545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173784.674544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173784.679527:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173784.679549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:3.1:1786173784.679549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00000080:00000010:2.1:1786173784.688537:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.688540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173784.688546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:2.1:1786173784.688549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.688550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173784.688557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:1.1:1786173784.691541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173784.691547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173784.691558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00000080:00000010:2.1:1786173784.701531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.701541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:3.1:1786173784.701541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:2.1:1786173784.701545:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:3.1:1786173784.701549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:3.1:1786173784.705533:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173784.705534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173784.705539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173784.705545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:2.1:1786173784.713533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173784.713540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173784.713549:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173784.714548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173784.714552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:1.1:1786173784.714554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:3.1:1786173784.721547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:3.1:1786173784.721555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173784.722544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:1.1:1786173784.725527:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173784.725535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173784.725536:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173784.725543:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173784.725546:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:3.1:1786173784.728530:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173784.728533:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:1.1:1786173784.728539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173784.728542:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.728546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173784.732548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173784.732549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:3.1:1786173784.735536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:3.1:1786173784.735541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173784.741528:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173784.741543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173784.746537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:3.1:1786173784.753531:0:103738:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173784.753534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00010000:00000010:1.1:1786173784.753545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:1.1:1786173784.753553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:2.1:1786173784.758532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173784.759539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173784.759544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00000080:00000010:2.1:1786173784.762529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173784.762544:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:3.1:1786173784.762549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:3.1:1786173784.762559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173784.763546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173784.766531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.766535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00010000:00000010:2.1:1786173784.766538:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173784.771531:0:2384:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:1.1:1786173784.771539:0:2384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173784.772541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173784.774532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:3.1:1786173784.774551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00000080:00000010:2.1:1786173784.778528:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.778538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173784.778546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173784.783539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173784.783541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173784.787533:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:3.1:1786173784.787534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:3.1:1786173784.787542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00010000:00000010:2.1:1786173784.791530:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173784.791536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173784.795534:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:1.1:1786173784.795537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173784.795545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:3.1:1786173784.799619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:2.1:1786173784.799653:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00000080:00000010:2.1:1786173784.809531:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.813530:0:92300:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173784.813537:0:92300:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0a00.
00010000:00000010:2.1:1786173784.813545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:3.1:1786173784.813548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:3.1:1786173784.813556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173784.817527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173784.817537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173784.817542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:1.1:1786173784.817551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00010000:00000010:1.1:1786173784.821543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.821549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00000080:00000010:2.1:1786173784.826532:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173784.826543:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:3.1:1786173784.826549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173784.834543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:1.1:1786173784.838542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:1.1:1786173784.838548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:1.1:1786173784.838552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173784.839547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00000080:00000010:2.1:1786173784.842541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:3.1:1786173784.842546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:2.1:1786173784.842553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19e00.
00010000:00000010:3.1:1786173784.842555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:2.1:1786173784.846525:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193900.
00010000:00000010:2.1:1786173784.846528:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:2.1:1786173784.846530:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5ea00.
00010000:00000010:2.1:1786173784.846532:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193f00.
00010000:00000010:2.1:1786173784.846534:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18400.
00000080:00000010:2.1:1786173784.850533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00000080:00000010:2.1:1786173784.850543:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff97d1ab4d2940.
00010000:00000010:2.1:1786173784.850550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:2.1:1786173784.850552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173784.854529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3d00.
00010000:00000010:2.1:1786173784.854537:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:2.1:1786173784.854540:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173784.854543:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3060.
00010000:00000010:2.1:1786173784.859531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:2.1:1786173784.859536:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00000080:00000010:2.1:1786173784.859540:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca34a0.
00000080:00000010:2.1:1786173784.859546:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173784.863541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:2.1:1786173784.867534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:2.1:1786173784.867539:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173784.877594:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:1.1:1786173784.877601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00000080:00000010:2.1:1786173784.881529:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3f40.
00010000:00000010:2.1:1786173784.889544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:2.1:1786173784.889548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00000080:00000010:2.1:1786173784.894584:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173784.894596:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:2.1:1786173784.894598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:3.1:1786173784.903536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:3.1:1786173784.903542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173784.907535:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3f40.
00010000:00000010:2.1:1786173784.907547:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193f00.
00010000:00000010:2.1:1786173784.907552:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173784.912550:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff97d1b33fdac0.
00000080:00000010:2.1:1786173784.912558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173784.912564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:2.1:1786173784.912568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173784.912573:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173784.917534:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193f00.
00010000:00000010:2.1:1786173784.917540:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173784.917543:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3f40.
00000080:00000010:2.1:1786173784.922527:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3ca0.
00010000:00000010:2.1:1786173784.926526:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:2.1:1786173784.926531:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173784.930539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:2.1:1786173784.930548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193f00.
00010000:00000010:2.1:1786173784.930551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:2.1:1786173784.930553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173784.934534:0:103737:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff97d1b33fdac0.
00000080:00000010:2.1:1786173784.934542:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3f40.
00010000:00000010:2.1:1786173784.942501:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:2.1:1786173784.942507:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00000080:00000010:2.1:1786173784.942525:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3ca0.
00010000:00000010:2.1:1786173784.942531:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193400.
00010000:00000010:2.1:1786173784.942533:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00000080:00000010:2.1:1786173784.942536:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173784.946529:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173784.950537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0c00.
00010000:00000010:1.1:1786173784.950541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0800.
00010000:00000010:3.1:1786173784.957545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193100.
00010000:00000010:3.1:1786173784.957553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0200.
00010000:00000010:1.1:1786173784.961533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:1.1:1786173784.961537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1c00.
00010000:00000010:1.1:1786173784.965539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173784.965546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0000.
00010000:00000010:2.1:1786173784.965549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193f00.
00010000:00000010:2.1:1786173784.965555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173784.968571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00000080:00000010:2.1:1786173784.968581:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff97d1b33fdac0.
00010000:00000010:2.1:1786173784.977536:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:2.1:1786173784.977545:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173784.977550:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3860.
00010000:00000010:2.1:1786173784.983535:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:2.1:1786173784.983544:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173784.983551:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3ca0.
00010000:00000010:2.1:1786173784.983558:0:103737:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193f00.
00010000:00000010:2.1:1786173784.983563:0:103737:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173784.983567:0:103737:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3540.
00010000:00000010:1.1:1786173784.991530:0:103798:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193800.
00010000:00000010:1.1:1786173784.991539:0:103798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1a00.
00010000:00000010:2.1:1786173784.992532:0:61981:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173785.012535:0:2392:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a79b3400.
00010000:00000010:1.1:1786173785.012542:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a8962c00.
00000080:00000010:1.1:1786173785.012548:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aabd86e0.
00010000:00000010:1.1:1786173785.017532:0:72317:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a79b3b00.
00010000:00000010:1.1:1786173785.017542:0:72317:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a8963e00.
00000080:00000010:1.1:1786173785.017551:0:72317:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aabd8c80.
00010000:00000010:1.1:1786173785.048528:0:72317:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a79b3900.
00010000:00000010:1.1:1786173785.048533:0:72317:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a8963000.
00000080:00000010:1.1:1786173785.048540:0:72317:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aabd80e0.
00010000:00000010:1.1:1786173785.052541:0:103797:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9b300.
00010000:00000010:1.1:1786173785.052550:0:103797:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5e800.
00000080:00000010:1.1:1786173785.052560:0:103797:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192a1a0.
00010000:00000010:1.1:1786173785.059534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065900.
00010000:00000010:1.1:1786173785.059539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18e00.
00000080:00000010:1.1:1786173785.059541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0a60.
00010000:00000010:1.1:1786173785.063538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a79b3900.
00010000:00000010:1.1:1786173785.063542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a8963e00.
00000080:00000010:1.1:1786173785.063545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aabd8780.
00010000:00000010:3.1:1786173785.065539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bc00.
00010000:00000010:0.1:1786173785.065541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065600.
00010000:00000010:3.1:1786173785.065548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f800.
00010000:00000010:0.1:1786173785.065549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18400.
00000080:00000010:3.1:1786173785.065554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192ad00.
00000080:00000010:0.1:1786173785.065556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0b40.
00010000:00000010:2.1:1786173785.066547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:2.1:1786173785.066555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173785.066561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3820.
00010000:00000010:0.1:1786173785.070553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065d00.
00010000:00000010:0.1:1786173785.070558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19e00.
00000080:00000010:0.1:1786173785.070562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0e60.
00010000:00000010:2.1:1786173785.074534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193f00.
00010000:00000010:2.1:1786173785.074539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173785.074542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3300.
00010000:00000010:3.1:1786173785.074556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bc00.
00010000:00000010:3.1:1786173785.074563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f800.
00000080:00000010:3.1:1786173785.074567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192ad00.
00010000:00000010:2.1:1786173785.083535:0:103786:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:2.1:1786173785.083542:0:103786:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0e00.
00000080:00000010:2.1:1786173785.083548:0:103786:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3820.
00010000:00000010:2.1:1786173785.087554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:2.1:1786173785.087562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173785.087568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3e40.
00010000:00000010:2.1:1786173785.091536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193200.
00010000:00000010:0.1:1786173785.091539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065d00.
00010000:00000010:2.1:1786173785.091543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1e00.
00010000:00000010:0.1:1786173785.091545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19e00.
00000080:00000010:2.1:1786173785.091549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3120.
00000080:00000010:0.1:1786173785.091550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0e60.
00010000:00000010:2.1:1786173785.091561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193f00.
00010000:00000010:0.1:1786173785.091562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065600.
00010000:00000010:0.1:1786173785.091572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18400.
00010000:00000010:2.1:1786173785.091572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:0.1:1786173785.091577:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0b40.
00000080:00000010:2.1:1786173785.091580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3300.
00010000:00000010:0.1:1786173785.096529:0:103785:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065e00.
00010000:00000010:0.1:1786173785.096534:0:103785:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19a00.
00000080:00000010:0.1:1786173785.096537:0:103785:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0680.
00010000:00000010:2.1:1786173785.096544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:2.1:1786173785.096550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173785.096555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3760.
00010000:00000010:2.1:1786173785.100538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:0.1:1786173785.100545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065900.
00010000:00000010:2.1:1786173785.100546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00010000:00000010:3.1:1786173785.100549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bc00.
00000080:00000010:2.1:1786173785.100555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3e40.
00010000:00000010:0.1:1786173785.100558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18000.
00010000:00000010:3.1:1786173785.100559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f800.
00000080:00000010:0.1:1786173785.100567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0ac0.
00000080:00000010:3.1:1786173785.100568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192ad00.
00010000:00000010:0.1:1786173785.104550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065600.
00010000:00000010:0.1:1786173785.104555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18400.
00000080:00000010:0.1:1786173785.104558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0b40.
00010000:00000010:3.1:1786173785.113530:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bf00.
00010000:00000010:3.1:1786173785.113537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f400.
00000080:00000010:3.1:1786173785.113540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192a3c0.
00010000:00000010:2.1:1786173785.113557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:2.1:1786173785.113565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173785.113568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3760.
00010000:00000010:0.1:1786173785.117544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065900.
00010000:00000010:2.1:1786173785.117546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a79b3900.
00010000:00000010:0.1:1786173785.117550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19a00.
00010000:00000010:2.1:1786173785.117552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a8963c00.
00000080:00000010:0.1:1786173785.117554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0680.
00000080:00000010:2.1:1786173785.117558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aabd8ee0.
00010000:00000010:3.1:1786173785.121545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bc00.
00010000:00000010:3.1:1786173785.121553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f800.
00000080:00000010:3.1:1786173785.121561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192ad00.
00010000:00000010:3.1:1786173785.121569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a79b3400.
00010000:00000010:3.1:1786173785.121577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a8963600.
00010000:00000010:2.1:1786173785.121592:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:2.1:1786173785.121599:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173785.121604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3e40.
00010000:00000010:2.1:1786173785.121612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a79b3b00.
00010000:00000010:2.1:1786173785.121618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a8962000.
00000080:00000010:2.1:1786173785.121622:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aabd8ea0.
00000080:00000010:3.1:1786173785.126529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aabd80c0.
00010000:00000010:2.1:1786173785.126541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:2.1:1786173785.126546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173785.126549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3760.
00010000:00000010:3.1:1786173785.131540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bf00.
00010000:00000010:0.1:1786173785.131541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065900.
00010000:00000010:3.1:1786173785.131547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f400.
00010000:00000010:0.1:1786173785.131548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19a00.
00000080:00000010:0.1:1786173785.131556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0680.
00000080:00000010:3.1:1786173785.131556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192a3c0.
00010000:00000010:2.1:1786173785.135533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a79b3200.
00010000:00000010:2.1:1786173785.135539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a8963400.
00000080:00000010:2.1:1786173785.135542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aabd8400.
00010000:00000010:3.1:1786173785.135548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bc00.
00010000:00000010:3.1:1786173785.135555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f800.
00000080:00000010:3.1:1786173785.135558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192ad00.
00010000:00000010:2.1:1786173785.139531:0:103812:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193300.
00010000:00000010:2.1:1786173785.139538:0:103812:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1000.
00000080:00000010:2.1:1786173785.139544:0:103812:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3e40.
00010000:00000010:0.1:1786173785.139546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065600.
00010000:00000010:2.1:1786173785.139550:0:103812:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:0.1:1786173785.139555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18600.
00010000:00000010:2.1:1786173785.139557:0:103812:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:0.1:1786173785.139561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b06c0.
00000080:00000010:2.1:1786173785.139564:0:103812:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3760.
00010000:00000010:0.1:1786173785.143533:0:103810:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a79b3b00.
00010000:00000010:0.1:1786173785.143539:0:103810:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a8963e00.
00010000:00000010:3.1:1786173785.143544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bf00.
00000080:00000010:0.1:1786173785.143546:0:103810:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aabd89a0.
00010000:00000010:3.1:1786173785.143553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f400.
00000080:00000010:3.1:1786173785.143558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192a3c0.
00010000:00000010:2.1:1786173785.146535:0:103798:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193f00.
00010000:00000010:2.1:1786173785.146541:0:103798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173785.146546:0:103798:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3300.
00010000:00000010:2.1:1786173785.154539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a79b3400.
00010000:00000010:2.1:1786173785.154545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a8963800.
00000080:00000010:2.1:1786173785.154548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aabd87c0.
00010000:00000010:0.1:1786173785.154578:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065600.
00010000:00000010:0.1:1786173785.154583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19800.
00000080:00000010:0.1:1786173785.154587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0a20.
00010000:00000010:2.1:1786173785.159548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:3.1:1786173785.159548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bf00.
00010000:00000010:2.1:1786173785.159559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:3.1:1786173785.159559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f400.
00000080:00000010:2.1:1786173785.159569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3760.
00000080:00000010:3.1:1786173785.159570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192a3c0.
00010000:00000010:2.1:1786173785.164542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065900.
00010000:00000010:2.1:1786173785.164550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19c00.
00000080:00000010:2.1:1786173785.164556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0dc0.
00010000:00000010:2.1:1786173785.164648:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065e00.
00010000:00000010:2.1:1786173785.164653:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19200.
00000080:00000010:2.1:1786173785.164659:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0480.
00010000:00000010:3.1:1786173785.172538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bc00.
00010000:00000010:3.1:1786173785.172545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f800.
00000080:00000010:3.1:1786173785.172551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192ad00.
00010000:00000010:2.1:1786173785.172637:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065600.
00010000:00000010:2.1:1786173785.172643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19800.
00000080:00000010:2.1:1786173785.172646:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0a20.
00010000:00000010:2.1:1786173785.177541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:2.1:1786173785.177549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173785.177555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3720.
00010000:00000010:2.1:1786173785.177636:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065d00.
00010000:00000010:2.1:1786173785.177642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18600.
00000080:00000010:2.1:1786173785.177644:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0cc0.
00010000:00000010:2.1:1786173785.180631:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065c00.
00010000:00000010:2.1:1786173785.180636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19a00.
00000080:00000010:2.1:1786173785.180639:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0720.
00010000:00000010:2.1:1786173785.180724:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065a00.
00010000:00000010:2.1:1786173785.180731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18200.
00000080:00000010:2.1:1786173785.180837:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b01c0.
00010000:00000010:2.1:1786173785.180891:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065000.
00010000:00000010:2.1:1786173785.180894:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18a00.
00000080:00000010:2.1:1786173785.180896:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0ec0.
00010000:00000010:2.1:1786173785.182543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065300.
00010000:00000010:1.1:1786173785.182546:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065400.
00010000:00000010:2.1:1786173785.182550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19e00.
00010000:00000010:1.1:1786173785.182555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18000.
00000080:00000010:2.1:1786173785.182558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0120.
00000080:00000010:1.1:1786173785.182560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0260.
00010000:00000010:2.1:1786173785.182701:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065700.
00010000:00000010:2.1:1786173785.182705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18400.
00000080:00000010:2.1:1786173785.182708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0ea0.
00010000:00000010:3.1:1786173785.185548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bc00.
00010000:00000010:3.1:1786173785.185555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f800.
00000080:00000010:3.1:1786173785.185560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192ad00.
00010000:00000010:3.1:1786173785.189555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065800.
00010000:00000010:3.1:1786173785.189561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19400.
00000080:00000010:3.1:1786173785.189564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0460.
00010000:00000010:2.1:1786173785.193535:0:103789:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:3.1:1786173785.193535:0:103808:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bf00.
00010000:00000010:3.1:1786173785.193544:0:103808:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f400.
00010000:00000010:2.1:1786173785.193546:0:103789:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:3.1:1786173785.193551:0:103808:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192a3c0.
00000080:00000010:2.1:1786173785.193554:0:103789:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3660.
00010000:00000010:2.1:1786173785.198550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193f00.
00010000:00000010:2.1:1786173785.198557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173785.198561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca37c0.
00010000:00000010:3.1:1786173785.206541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bc00.
00010000:00000010:3.1:1786173785.206549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f800.
00000080:00000010:3.1:1786173785.206555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192ad00.
00010000:00000010:2.1:1786173785.207534:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065200.
00010000:00000010:2.1:1786173785.207540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18400.
00000080:00000010:2.1:1786173785.207543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0de0.
00010000:00000010:2.1:1786173785.207603:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065500.
00010000:00000010:2.1:1786173785.207605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18000.
00000080:00000010:2.1:1786173785.207607:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0f60.
00010000:00000010:2.1:1786173785.208529:0:103802:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193e00.
00010000:00000010:2.1:1786173785.208536:0:103802:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00000080:00000010:2.1:1786173785.208542:0:103802:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3ba0.
00010000:00000010:2.1:1786173785.208629:0:103802:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065b00.
00010000:00000010:2.1:1786173785.208634:0:103802:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19e00.
00000080:00000010:2.1:1786173785.208641:0:103802:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0940.
00010000:00000010:1.1:1786173785.211538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a79b3900.
00010000:00000010:1.1:1786173785.211546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a8962600.
00010000:00000010:3.1:1786173785.211548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9bf00.
00000080:00000010:1.1:1786173785.211551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aabd8f80.
00010000:00000010:3.1:1786173785.211556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5f400.
00000080:00000010:3.1:1786173785.211562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192a3c0.
00010000:00000010:2.1:1786173785.211670:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065100.
00010000:00000010:2.1:1786173785.211676:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18a00.
00000080:00000010:2.1:1786173785.211680:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0660.
00010000:00000010:3.1:1786173785.213550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9b000.
00010000:00000010:3.1:1786173785.213555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5ee00.
00000080:00000010:3.1:1786173785.213558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192ad80.
00010000:00000010:2.1:1786173785.217544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193f00.
00010000:00000010:2.1:1786173785.217552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173785.217558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3e20.
00010000:00000010:2.1:1786173785.223548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065b00.
00010000:00000010:2.1:1786173785.223558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18600.
00000080:00000010:2.1:1786173785.223564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0880.
00010000:00000010:2.1:1786173785.225536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065500.
00010000:00000010:3.1:1786173785.225542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9b000.
00010000:00000010:2.1:1786173785.225543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19800.
00000080:00000010:2.1:1786173785.225551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0600.
00010000:00000010:3.1:1786173785.225554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5ee00.
00000080:00000010:3.1:1786173785.225562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192ad80.
00010000:00000010:2.1:1786173785.233542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193f00.
00010000:00000010:2.1:1786173785.233552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1600.
00000080:00000010:2.1:1786173785.233561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca38e0.
00010000:00000010:2.1:1786173785.233642:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065200.
00010000:00000010:2.1:1786173785.233649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19c00.
00000080:00000010:2.1:1786173785.233654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0040.
00010000:00000010:2.1:1786173785.247542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193b00.
00010000:00000010:2.1:1786173785.247549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1200.
00000080:00000010:0.1:1786173785.256566:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff97d1b33fdac0.
00000080:00000010:0.1:1786173785.256575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca31a0.
00010000:00000010:0.1:1786173785.256584:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065800.
00010000:00000010:0.1:1786173785.256588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a77c4000.
00000080:00000010:0.1:1786173785.256593:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0c20.
00010000:00000010:0.1:1786173785.261531:0:103819:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065700.
00010000:00000010:0.1:1786173785.261539:0:103819:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a77c5a00.
00000080:00000010:0.1:1786173785.261546:0:103819:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0ca0.
00010000:00000010:0.1:1786173785.261552:0:103819:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065400.
00010000:00000010:0.1:1786173785.261558:0:103819:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a77c4a00.
00000080:00000010:0.1:1786173785.261562:0:103819:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b05c0.
00010000:00000010:0.1:1786173785.266554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065300.
00010000:00000010:0.1:1786173785.266561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a77c5c00.
00000080:00000010:0.1:1786173785.266566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0040.
00010000:00000010:3.1:1786173787.544582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5ea00.
00000080:00000010:3.1:1786173787.544599:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192a280.
00010000:00000010:3.1:1786173787.544612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193000.
00010000:00000010:3.1:1786173787.544620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aaed3000.
00000800:00000200:3.1:1786173848.952609:0:2383:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 6 ksnr_max_conns 1
00000800:00000200:3.1:1786173848.952615:0:2383:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:3.1:1786173848.952618:0:2383:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1
00000800:00000200:2.1:1786173852.153889:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173852.154167:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173852.154297:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173852.154423:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173857.209266:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173857.209736:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173857.209975:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173857.210189:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173862.393153:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173862.393412:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173862.393604:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173862.393769:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173867.513074:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173867.513078:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 4, 2 secs later)
00000800:00000200:2.1:1786173867.513556:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173867.513559:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 4, 2 secs later)
00000800:00000200:2.1:1786173867.513885:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173867.513888:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 4, 2 secs later)
00000800:00000200:2.1:1786173867.514188:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173867.514191:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 4, 2 secs later)
00000800:00000200:2.1:1786173872.633366:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173872.633954:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173872.634060:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173872.634163:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173873.641468:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173873.641478:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 7 secs later)
00000800:00000200:2.1:1786173873.642604:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173873.642608:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 7 secs later)
00000800:00000200:2.1:1786173873.643013:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173873.643016:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 7 secs later)
00000800:00000200:2.1:1786173873.643264:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173873.643265:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 7 secs later)
00000800:00000200:2.1:1786173873.643444:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173873.643446:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 7 secs later)
00000800:00000200:2.1:1786173878.777478:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173878.777486:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 2 secs later)
00000800:00000200:2.1:1786173878.778288:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173878.778291:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 2 secs later)
00000800:00000200:2.1:1786173878.778596:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173878.778598:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 2 secs later)
00000800:00000200:2.1:1786173878.778793:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173878.778794:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 2 secs later)
00000800:00000200:2.1:1786173878.779004:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173878.779006:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 8, 2 secs later)
00000800:00000200:2.1:1786173883.897043:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173883.897322:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173883.897499:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173883.897667:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786173883.897754:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173889.017839:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173889.017842:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 14 secs later)
00000800:00000200:1.1:1786173889.018192:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173889.018194:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 14 secs later)
00000800:00000200:1.1:1786173889.018481:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173889.018484:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 14 secs later)
00000800:00000200:1.1:1786173889.018728:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173889.018730:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 14 secs later)
00000800:00000200:1.1:1786173889.018925:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173889.018927:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 14 secs later)
00000800:00000200:3.1:1786173894.137466:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173894.137477:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 9 secs later)
00000800:00000200:3.1:1786173894.138953:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173894.138956:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 9 secs later)
00000800:00000200:3.1:1786173894.139182:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173894.139184:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 9 secs later)
00000800:00000200:3.1:1786173894.139371:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173894.139373:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 9 secs later)
00000800:00000200:3.1:1786173894.139660:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173894.139662:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 9 secs later)
00000800:00000200:3.1:1786173899.257460:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173899.257468:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 4 secs later)
00000800:00000200:3.1:1786173899.258699:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173899.258702:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 4 secs later)
00000800:00000200:3.1:1786173899.258919:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173899.258920:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 4 secs later)
00000800:00000200:3.1:1786173899.259098:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173899.259100:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 4 secs later)
00000800:00000200:3.1:1786173899.259513:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173899.259516:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 16, 4 secs later)
00000800:00000200:1.1:1786173904.377242:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173904.377730:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173904.377939:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173904.378113:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173904.378245:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173909.497280:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173909.497286:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 30 secs later)
00000800:00000200:1.1:1786173909.498178:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173909.498180:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 30 secs later)
00000800:00000200:1.1:1786173909.498448:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173909.498450:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 30 secs later)
00000800:00000200:1.1:1786173909.498782:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173909.498785:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 30 secs later)
00000800:00000200:1.1:1786173909.499170:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173909.499172:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 30 secs later)
00000800:00000200:3.1:1786173914.617272:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173914.617281:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 24 secs later)
00000800:00000200:3.1:1786173914.617976:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173914.617978:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 24 secs later)
00000800:00000200:3.1:1786173914.618302:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173914.618305:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 24 secs later)
00000800:00000200:3.1:1786173914.618552:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173914.618555:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 24 secs later)
00000800:00000200:3.1:1786173914.618735:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173914.618737:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 24 secs later)
00000800:00000200:3.1:1786173919.737403:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173919.737412:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 19 secs later)
00000800:00000200:3.1:1786173919.738280:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173919.738283:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 19 secs later)
00000800:00000200:3.1:1786173919.738732:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173919.738736:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 19 secs later)
00000800:00000200:3.1:1786173919.738997:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173919.739000:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 19 secs later)
00000800:00000200:3.1:1786173919.739272:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173919.739274:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 19 secs later)
00000800:00000200:3.1:1786173924.857298:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173924.857305:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 14 secs later)
00000800:00000200:3.1:1786173924.858211:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173924.858213:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 14 secs later)
00000800:00000200:3.1:1786173924.858791:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173924.858795:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 14 secs later)
00000800:00000200:3.1:1786173924.859132:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173924.859135:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 14 secs later)
00000800:00000200:3.1:1786173924.859450:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173924.859453:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 14 secs later)
00000800:00000200:1.1:1786173929.977266:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173929.977272:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 9 secs later)
00000800:00000200:1.1:1786173929.977938:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173929.977940:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 9 secs later)
00000800:00000200:1.1:1786173929.978180:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173929.978181:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 9 secs later)
00000800:00000200:1.1:1786173929.978419:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173929.978421:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 9 secs later)
00000800:00000200:1.1:1786173929.978651:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173929.978654:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 9 secs later)
00000800:00000200:1.1:1786173935.097223:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173935.097228:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 4 secs later)
00000800:00000200:1.1:1786173935.097814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173935.097816:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 4 secs later)
00000800:00000200:1.1:1786173935.098056:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173935.098058:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 4 secs later)
00000800:00000200:1.1:1786173935.098284:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173935.098287:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 4 secs later)
00000800:00000200:1.1:1786173935.098548:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173935.098550:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 32, 4 secs later)
00000080:00000010:1.1:1786173938.750549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3680.
00000080:00000010:1.1:1786173938.750571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca33c0.
00000080:00000010:1.1:1786173938.754538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3f00.
00000080:00000010:1.1:1786173938.754544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1aaca3c40.
00000080:00000010:1.1:1786173938.754547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1b192a8e0.
00000080:00000010:1.1:1786173938.754551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff97d1ba0b0740.
00010000:00000010:1.1:1786173938.754554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193700.
00010000:00000010:1.1:1786173938.754557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0600.
00010000:00000010:1.1:1786173938.754560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065600.
00010000:00000010:1.1:1786173938.754562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a77c4200.
00010000:00000010:1.1:1786173938.754564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193c00.
00010000:00000010:1.1:1786173938.754566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e0400.
00010000:00000010:1.1:1786173938.754568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193600.
00010000:00000010:1.1:1786173938.754570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1800.
00010000:00000010:1.1:1786173938.754571:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193a00.
00010000:00000010:1.1:1786173938.754573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aa8e1400.
00010000:00000010:1.1:1786173938.754574:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193500.
00010000:00000010:1.1:1786173938.754576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aaed2a00.
00010000:00000010:1.1:1786173938.754578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5fe00.
00010000:00000010:1.1:1786173938.754580:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186065f00.
00010000:00000010:1.1:1786173938.754582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a19600.
00010000:00000010:1.1:1786173938.754584:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1b2cc6400.
00010000:00000010:1.1:1786173938.754586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aaed2000.
00010000:00000010:1.1:1786173938.754587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aaed2c00.
00010000:00000010:1.1:1786173938.754589:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1b2cc6500.
00010000:00000010:1.1:1786173938.754590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aaed2600.
00010000:00000010:1.1:1786173938.754592:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1b2cc6b00.
00010000:00000010:1.1:1786173938.754595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aaed2200.
00010000:00000010:1.1:1786173938.754596:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1b2cc6300.
00010000:00000010:1.1:1786173938.754598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a9a18c00.
00000800:00000200:1.1:1786173940.217093:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173940.217409:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173940.217786:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173940.218043:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173940.218194:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173945.338085:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173945.338090:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786173945.338504:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173945.338506:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786173945.338875:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173945.338878:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786173945.339084:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173945.339085:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786173945.339378:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173945.339380:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786173950.457121:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173950.457126:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786173950.457846:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173950.457848:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786173950.458037:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173950.458038:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786173950.458229:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173950.458231:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786173950.458478:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173950.458480:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786173955.577304:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173955.577310:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786173955.577869:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173955.577870:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786173955.578059:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173955.578061:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786173955.578288:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173955.578290:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786173955.578560:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173955.578562:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786173960.697211:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173960.697217:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786173960.697853:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173960.697855:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786173960.698071:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173960.698072:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786173960.698261:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173960.698263:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786173960.698443:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173960.698445:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786173965.817093:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173965.817100:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786173965.817792:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173965.817794:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786173965.817978:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173965.817980:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786173965.818193:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173965.818196:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786173965.818421:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173965.818424:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786173970.937209:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173970.937214:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786173970.937978:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173970.937980:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786173970.938214:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173970.938215:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786173970.938395:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173970.938396:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786173970.938819:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173970.938821:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786173976.057086:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173976.057091:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786173976.057807:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173976.057809:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786173976.058019:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173976.058021:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786173976.058233:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173976.058235:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786173976.058414:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173976.058415:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786173981.177258:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173981.177265:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786173981.178207:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173981.178209:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786173981.178400:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173981.178402:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786173981.178631:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173981.178633:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786173981.178923:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173981.178926:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786173986.297226:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173986.297232:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786173986.297924:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173986.297926:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786173986.298118:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173986.298120:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786173986.298335:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173986.298337:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786173986.298605:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173986.298606:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786173991.417241:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173991.417249:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786173991.417963:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173991.417966:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786173991.418253:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173991.418255:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786173991.418537:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173991.418540:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786173991.418940:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786173991.418942:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786173996.538670:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173996.538689:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786173996.539079:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173996.539081:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786173996.539317:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173996.539319:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786173996.539575:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173996.539577:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786173996.539782:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786173996.539785:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174001.658358:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174001.658364:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174001.659048:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174001.659050:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174001.659280:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174001.659282:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174001.659456:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174001.659458:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174001.659737:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174001.659739:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786174006.777249:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174006.777679:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174006.777839:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174006.777985:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174006.778209:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174011.897428:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174011.897431:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174011.898080:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174011.898083:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174011.898413:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174011.898415:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174011.899585:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174011.899587:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174011.899892:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174011.899895:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174017.017118:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174017.017123:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174017.017778:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174017.017780:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174017.017978:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174017.017982:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174017.018200:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174017.018202:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174017.018490:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174017.018491:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174022.137073:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174022.137078:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174022.137815:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174022.137818:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174022.138080:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174022.138082:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174022.138309:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174022.138310:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174022.138508:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174022.138525:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174027.257448:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174027.257459:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174027.258153:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174027.258155:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174027.258360:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174027.258363:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174027.258719:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174027.258722:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174027.258960:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174027.258963:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174032.377138:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174032.377145:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174032.377848:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174032.377850:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174032.378043:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174032.378045:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174032.378221:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174032.378222:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174032.378489:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174032.378491:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174037.497492:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174037.497499:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174037.498427:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174037.498429:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174037.498730:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174037.498732:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174037.498945:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174037.498947:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174037.499124:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174037.499126:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174042.617236:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174042.617246:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174042.617863:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174042.617865:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174042.618125:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174042.618127:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174042.618368:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174042.618370:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174042.618607:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174042.618610:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174047.737240:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174047.737246:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174047.737941:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174047.737945:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174047.738343:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174047.738344:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174047.738612:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174047.738614:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174047.738800:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174047.738801:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174052.857175:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174052.857180:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174052.857858:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174052.857861:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174052.858214:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174052.858216:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174052.858563:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174052.858567:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174052.858890:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174052.858893:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174057.977706:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174057.977716:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174057.978354:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174057.978356:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174057.978613:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174057.978615:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174057.978870:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174057.978872:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174057.979097:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174057.979100:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174063.097307:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174063.097314:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174063.097854:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174063.097856:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174063.098041:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174063.098043:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174063.098252:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174063.098254:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174063.098469:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174063.098471:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174068.217786:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174068.217804:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174068.218990:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174068.218994:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174068.219508:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174068.219542:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174068.220042:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174068.220045:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174068.220498:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174068.220503:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00010000:00000010:2.1:1786174068.798566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a79b3100.
00010000:00000010:2.1:1786174068.798575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1a8963a00.
00000800:00000200:3.1:1786174073.337283:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174073.337653:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174073.337912:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174073.338132:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174073.338274:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174078.457301:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174078.457308:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174078.458021:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174078.458024:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174078.458295:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174078.458297:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174078.458510:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174078.458533:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174078.458774:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174078.458775:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174083.577305:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174083.577312:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786174083.577976:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174083.577979:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786174083.578299:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174083.578302:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786174083.578614:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174083.578617:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786174083.578934:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174083.578936:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786174088.697185:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174088.697190:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786174088.697929:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174088.697931:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786174088.698181:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174088.698183:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786174088.698363:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174088.698365:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786174088.698677:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174088.698679:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786174093.817182:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174093.817188:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786174093.817781:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174093.817783:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786174093.817973:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174093.817976:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786174093.818209:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174093.818211:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786174093.818456:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174093.818458:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786174098.937342:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174098.937351:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174098.938363:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174098.938365:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174098.938675:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174098.938677:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174098.938921:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174098.938922:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174098.939141:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174098.939144:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174104.057728:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174104.057741:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174104.058717:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174104.058720:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174104.059115:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174104.059118:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174104.059458:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174104.059461:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174104.059895:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174104.059899:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174109.177437:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174109.177446:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174109.178296:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174109.178297:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174109.178490:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174109.178492:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174109.178727:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174109.178728:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174109.178987:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174109.178989:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174114.297390:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174114.297396:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174114.298320:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174114.298322:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174114.298629:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174114.298632:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174114.298958:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174114.298961:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174114.299255:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174114.299258:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174119.417277:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174119.417284:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174119.417949:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174119.417953:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174119.418406:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174119.418407:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174119.418736:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174119.418738:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174119.418959:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174119.418961:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174124.537225:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174124.537234:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174124.537958:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174124.537963:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174124.538327:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174124.538329:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174124.538652:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174124.538654:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174124.538862:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174124.538864:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174129.657285:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174129.657292:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1786174129.657997:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174129.658000:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1786174129.658318:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174129.658321:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1786174129.658686:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174129.658689:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1786174129.659052:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174129.659055:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1786174134.777171:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174134.777174:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786174134.777595:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174134.777597:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786174134.779126:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174134.779138:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786174134.779736:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174134.779739:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786174134.780010:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174134.780011:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786174139.897181:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174139.897533:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174139.897895:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174139.898272:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174139.898454:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00010000:00000010:3.1:1786174142.526571:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d186193d00.
00010000:00000010:3.1:1786174142.526585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aaed3e00.
00010000:00000010:3.1:1786174142.526590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff97d1a8b9b400.
00010000:00000010:3.1:1786174142.526595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97d1aab5ec00.
00000800:00000200:3.1:1786174145.018169:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174145.018174:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174145.018680:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174145.018683:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174145.018954:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174145.018956:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174145.019152:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174145.019155:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174145.019392:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174145.019394:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174150.137172:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174150.137177:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174150.137825:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174150.137829:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174150.138069:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174150.138071:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174150.138317:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174150.138319:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174150.138499:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174150.138501:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174155.257404:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174155.257438:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174155.258336:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174155.258342:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174155.258905:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174155.258907:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174155.259313:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174155.259315:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174155.259668:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174155.259670:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174160.377151:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174160.377157:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174160.377798:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174160.377800:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174160.377986:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174160.377988:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174160.378215:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174160.378217:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174160.378467:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174160.378469:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174165.497778:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174165.497793:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174165.499254:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174165.499257:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174165.499718:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174165.499722:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174165.500115:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174165.500132:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174165.500769:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174165.500773:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174170.617688:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174170.617705:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174170.619234:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174170.619239:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174170.619720:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174170.619724:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174170.619983:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174170.619985:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174170.620351:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174170.620353:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174175.737465:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174175.737472:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174175.738439:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174175.738451:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174175.738768:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174175.738770:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174175.739113:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174175.739116:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174175.739429:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174175.739432:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174180.857242:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174180.857252:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174180.857747:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174180.857749:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174180.857958:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174180.857959:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174180.858173:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174180.858175:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174180.858388:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174180.858390:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174185.977759:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174185.977770:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174185.978746:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174185.978749:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174185.979123:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174185.979125:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174185.979527:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174185.979530:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174185.979965:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174185.979968:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174191.097330:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174191.097336:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174191.098021:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174191.098023:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174191.098307:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174191.098310:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174191.098531:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174191.098533:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174191.098844:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174191.098848:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174196.217308:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174196.217313:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174196.217938:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174196.217940:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174196.218154:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174196.218157:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174196.218384:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174196.218387:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174196.218618:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174196.218620:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174201.337303:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174201.337309:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174201.337931:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174201.337932:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174201.338177:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174201.338179:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174201.338392:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174201.338393:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174201.338620:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174201.338622:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174206.457259:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174206.457732:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174206.457921:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174206.458086:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174206.458243:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174211.577344:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174211.577352:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:1.1:1786174211.578164:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174211.578167:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:1.1:1786174211.578399:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174211.578401:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:1.1:1786174211.578666:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174211.578668:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:1.1:1786174211.578981:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174211.578983:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:3.1:1786174216.697391:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174216.697395:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786174216.698073:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174216.698077:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786174216.698411:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174216.698413:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786174216.698685:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174216.698687:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786174216.698867:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174216.698869:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786174221.817214:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174221.817218:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174221.817865:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174221.817867:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174221.818104:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174221.818105:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174221.818316:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174221.818318:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174221.818672:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174221.818674:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174226.937435:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174226.937444:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174226.938412:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174226.938413:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174226.938651:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174226.938652:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174226.938838:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174226.938840:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174226.939151:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174226.939154:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174232.057875:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174232.057882:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174232.059329:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174232.059335:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174232.059981:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174232.059986:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174232.060377:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174232.060380:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174232.060810:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174232.060813:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174237.177304:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174237.177309:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174237.178022:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174237.178024:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174237.178229:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174237.178231:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174237.178459:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174237.178461:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174237.178750:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174237.178752:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174242.297473:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174242.297481:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174242.298584:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174242.298586:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174242.298816:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174242.298818:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174242.299075:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174242.299078:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174242.299300:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174242.299302:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174247.417743:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174247.417752:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174247.418825:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174247.418827:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174247.419230:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174247.419232:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174247.419660:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174247.419663:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174247.420048:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174247.420051:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174252.537624:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174252.537629:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174252.538443:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174252.538445:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174252.538822:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174252.538824:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174252.539132:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174252.539134:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174252.539452:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174252.539454:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174257.657619:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174257.657625:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786174257.658498:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174257.658501:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786174257.658797:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174257.658800:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786174257.659045:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174257.659048:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786174257.659320:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174257.659324:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1786174262.777185:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174262.777187:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174262.777901:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174262.777906:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174262.778158:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174262.778159:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174262.778400:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174262.778402:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174262.778630:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174262.778632:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174267.897334:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174267.897340:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174267.897994:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174267.897995:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174267.898229:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174267.898230:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174267.898424:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174267.898426:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174267.898732:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174267.898733:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174273.017613:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174273.018453:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174273.019176:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174273.019503:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174273.019828:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174278.137585:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174278.137594:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174278.138782:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174278.138795:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174278.139283:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174278.139287:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174278.139617:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174278.139619:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174278.139920:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174278.139922:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174283.257590:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174283.257597:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174283.258462:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174283.258465:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174283.258970:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174283.258972:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174283.259370:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174283.259373:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174283.259833:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174283.259837:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174288.377372:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174288.377382:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174288.378582:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174288.378584:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174288.378782:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174288.378783:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174288.378996:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174288.378998:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174288.379263:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174288.379265:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174293.497179:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174293.497188:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174293.497822:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174293.497824:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174293.498063:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174293.498065:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174293.498308:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174293.498310:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174293.498493:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174293.498494:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174298.617191:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174298.617199:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174298.617873:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174298.617875:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174298.618071:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174298.618073:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174298.618250:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174298.618252:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174298.618501:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174298.618503:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174303.737424:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174303.737429:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174303.738173:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174303.738175:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174303.738427:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174303.738429:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174303.738765:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174303.738767:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174303.739087:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174303.739090:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174308.857120:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174308.857125:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174308.857876:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174308.857881:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174308.858168:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174308.858170:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174308.858506:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174308.858508:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174308.858772:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174308.858774:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174313.977609:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174313.977622:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174313.978476:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174313.978479:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174313.978844:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174313.978846:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174313.979101:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174313.979103:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174313.979342:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174313.979345:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174319.097327:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174319.097331:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174319.098087:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174319.098088:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174319.098279:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174319.098280:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174319.098485:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174319.098487:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174319.098860:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174319.098861:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174324.217178:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174324.217182:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174324.217710:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174324.217712:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174324.217889:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174324.217891:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174324.218100:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174324.218102:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174324.218310:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174324.218312:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174329.337684:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174329.337690:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174329.338804:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174329.338806:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174329.339216:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174329.339218:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174329.339625:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174329.339628:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174329.339946:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174329.339949:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174334.458040:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174334.458058:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174334.459903:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174334.459906:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174334.460333:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174334.460337:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174334.460849:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174334.460854:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174334.461470:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174334.461474:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174339.577937:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174339.578748:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174339.579035:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174339.579364:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174339.579738:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174344.698643:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174344.698653:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174344.699616:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174344.699619:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174344.700078:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174344.700082:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174344.700496:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174344.700499:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174344.700944:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174344.700948:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174349.817666:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174349.817706:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174349.819119:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174349.819122:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174349.819597:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174349.819601:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174349.820036:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174349.820041:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174349.820662:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174349.820666:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174354.937119:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174354.937126:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174354.937730:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174354.937732:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174354.937951:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174354.937952:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174354.938133:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174354.938134:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174354.938326:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174354.938328:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174360.057895:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174360.057910:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174360.059138:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174360.059144:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174360.059696:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174360.059700:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174360.060063:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174360.060066:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174360.060601:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174360.060606:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174365.177295:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174365.177299:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174365.178436:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174365.178439:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174365.178857:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174365.178859:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174365.179105:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174365.179108:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174365.179334:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174365.179336:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174370.297891:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174370.297908:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174370.299911:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174370.299922:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174370.300712:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174370.300715:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174370.301189:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174370.301192:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174370.301523:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174370.301526:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174375.417141:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174375.417147:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174375.417788:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174375.417790:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174375.417990:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174375.417993:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174375.418230:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174375.418232:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174375.418504:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174375.418506:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174380.537229:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174380.537235:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:1.1:1786174380.538043:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174380.538048:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:1.1:1786174380.538469:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174380.538471:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:1.1:1786174380.538789:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174380.538791:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:1.1:1786174380.539015:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174380.539017:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:1.1:1786174385.657749:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174385.657770:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174385.659563:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174385.659566:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174385.659892:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174385.659906:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174385.660325:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174385.660327:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174385.660643:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174385.660645:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174390.777428:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174390.777434:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174390.778441:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174390.778444:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174390.778827:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174390.778830:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174390.779162:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174390.779164:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174390.779593:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174390.779596:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174395.897559:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174395.897566:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174395.898764:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174395.898773:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174395.899275:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174395.899280:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174395.899923:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174395.899926:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174395.900247:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174395.900250:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174401.017155:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174401.017161:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174401.017856:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174401.017858:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174401.018051:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174401.018052:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174401.018240:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174401.018242:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174401.018493:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174401.018495:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174406.137542:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174406.138435:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174406.138996:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174406.139173:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174406.139369:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174411.258311:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174411.258314:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174411.258715:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174411.258717:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174411.258963:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174411.258965:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174411.259165:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174411.259168:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174411.259358:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174411.259359:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174416.377915:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174416.377931:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174416.379442:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174416.379445:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174416.379925:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174416.379928:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174416.380280:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174416.380283:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174416.380881:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174416.380884:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174421.497415:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174421.497423:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174421.498393:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174421.498395:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174421.498632:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174421.498634:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174421.498829:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174421.498831:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174421.499093:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174421.499095:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174426.617775:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174426.617791:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786174426.618626:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174426.618629:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786174426.618890:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174426.618892:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786174426.619116:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174426.619120:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786174426.619389:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174426.619391:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786174431.737446:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174431.737452:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174431.738240:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174431.738241:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174431.738462:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174431.738464:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174431.738691:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174431.738693:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174431.738900:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174431.738902:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174436.857618:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174436.857625:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174436.858383:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174436.858385:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174436.858680:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174436.858682:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174436.859040:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174436.859042:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174436.859339:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174436.859341:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174441.977822:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174441.977839:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174441.979103:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174441.979107:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174441.979590:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174441.979593:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174441.980083:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174441.980087:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174441.980497:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174441.980503:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786174447.097443:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174447.097452:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174447.098530:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174447.098533:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174447.098748:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174447.098749:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174447.098961:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174447.098963:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174447.099246:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174447.099248:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174452.217349:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174452.217357:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174452.218161:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174452.218163:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174452.218409:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174452.218410:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174452.218729:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174452.218732:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174452.219021:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174452.219023:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174457.337642:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174457.337650:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174457.338438:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174457.338442:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174457.338855:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174457.338858:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174457.339210:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174457.339214:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174457.339629:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174457.339632:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174462.457276:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174462.457281:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174462.457950:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174462.457952:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174462.458161:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174462.458163:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174462.458392:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174462.458394:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174462.458727:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174462.458729:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174467.577175:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174467.577179:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786174467.578173:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174467.578175:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786174467.578410:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174467.578412:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786174467.578634:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174467.578636:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786174467.578911:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174467.578913:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786174472.697448:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174472.698190:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174472.698615:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174472.698975:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174472.699229:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174477.818317:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174477.818325:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174477.818646:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174477.818649:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174477.818936:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174477.818937:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174477.819148:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174477.819150:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174477.819327:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174477.819329:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174482.937530:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174482.937539:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174482.938408:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174482.938411:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174482.938715:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174482.938717:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174482.939038:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174482.939041:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174482.939348:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174482.939350:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174488.057851:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174488.057860:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174488.058837:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174488.058842:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174488.059386:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174488.059392:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174488.059989:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174488.059994:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174488.060569:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174488.060574:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174493.177183:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174493.177196:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174493.177890:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174493.177893:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174493.178157:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174493.178158:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174493.178418:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174493.178419:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174493.178650:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174493.178653:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174498.297389:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174498.297395:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174498.298119:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174498.298122:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174498.298484:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174498.298487:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174498.298908:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174498.298911:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174498.299269:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174498.299273:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174503.417444:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174503.417454:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174503.418320:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174503.418324:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174503.418789:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174503.418793:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174503.419267:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174503.419270:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174503.419717:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174503.419721:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786174508.537254:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174508.537261:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786174508.537998:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174508.538001:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786174508.538296:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174508.538298:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786174508.538581:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174508.538583:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786174508.538854:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174508.538856:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786174513.658041:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174513.658063:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174513.659458:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174513.659462:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174513.659845:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174513.659849:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174513.660321:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174513.660324:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174513.660647:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174513.660650:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786174518.777223:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174518.777233:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174518.777982:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174518.777984:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174518.778240:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174518.778243:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174518.778476:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174518.778478:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174518.778831:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174518.778834:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174523.897942:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174523.897957:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174523.899359:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174523.899363:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174523.899752:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174523.899754:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174523.899985:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174523.899988:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174523.900336:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174523.900338:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174529.017313:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174529.017319:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174529.017871:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174529.017873:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174529.018072:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174529.018073:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174529.018325:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174529.018327:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174529.018544:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174529.018546:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174534.137794:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174534.137806:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174534.138906:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174534.138910:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174534.139306:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174534.139309:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174534.139701:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174534.139704:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174534.140124:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174534.140128:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174539.257268:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174539.257731:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174539.257948:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174539.258108:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174539.258260:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174544.377170:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174544.377177:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174544.377959:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174544.377966:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174544.378300:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174544.378303:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174544.378630:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174544.378632:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174544.378846:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174544.378847:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174549.497578:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174549.497582:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174549.499108:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174549.499115:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174549.499865:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174549.499867:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174549.500108:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174549.500110:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174549.500287:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174549.500289:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174554.617244:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174554.617252:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174554.617904:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174554.617907:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174554.618226:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174554.618228:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174554.618590:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174554.618593:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174554.618899:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174554.618901:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174559.737698:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174559.737714:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174559.739000:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174559.739004:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174559.739461:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174559.739465:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174559.739858:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174559.739861:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174559.740492:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174559.740496:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174564.857663:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174564.857670:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174564.859126:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174564.859130:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174564.859603:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174564.859606:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174564.860059:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174564.860062:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174564.860681:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174564.860687:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174569.977779:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174569.977793:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174569.978986:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174569.978990:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174569.979580:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174569.979586:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174569.980056:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174569.980059:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174569.980454:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174569.980457:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174575.097888:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174575.097899:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174575.099704:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174575.099707:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174575.099991:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174575.099993:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174575.100276:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174575.100280:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174575.100719:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174575.100722:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174580.217437:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174580.217441:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174580.219619:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174580.219622:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174580.219893:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174580.219895:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174580.220214:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174580.220216:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174580.220478:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174580.220481:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174585.337276:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174585.337283:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174585.338599:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174585.338604:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174585.338858:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174585.338860:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174585.339186:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174585.339188:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174585.339517:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174585.339519:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174590.457487:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174590.457502:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174590.458483:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174590.458486:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174590.458821:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174590.458823:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174590.459081:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174590.459083:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174590.459515:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174590.459557:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174595.577688:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174595.577700:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174595.579182:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174595.579188:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174595.580024:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174595.580026:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174595.580474:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174595.580477:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174595.580782:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174595.580784:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174600.697096:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174600.697100:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174600.697491:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174600.697493:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174600.697716:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174600.697718:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174600.697920:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174600.697922:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174600.698102:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174600.698103:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174605.817605:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174605.818269:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174605.818882:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174605.819133:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174605.820606:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174610.937499:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174610.937568:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174610.938674:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174610.938688:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174610.939086:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174610.939088:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174610.939484:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174610.939490:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174610.940112:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174610.940114:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174616.057507:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174616.057545:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174616.058550:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174616.058552:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174616.058797:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174616.058800:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174616.058988:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174616.058990:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174616.059197:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174616.059199:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174621.177963:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174621.177975:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174621.179393:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174621.179397:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174621.179932:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174621.179935:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174621.180323:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174621.180327:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174621.181016:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174621.181020:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174626.298235:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174626.298248:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174626.299681:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174626.299684:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174626.300135:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174626.300139:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174626.300582:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174626.300587:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174626.301073:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174626.301079:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174631.417497:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174631.417541:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174631.418966:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174631.418974:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174631.419491:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174631.419493:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174631.419809:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174631.419811:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174631.419990:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174631.419992:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174636.537427:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174636.537432:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174636.538694:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174636.538702:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174636.539152:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174636.539155:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174636.539611:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174636.539613:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174636.539940:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174636.539943:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174641.657621:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174641.657631:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174641.658813:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174641.658816:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174641.659218:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174641.659223:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174641.659687:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174641.659690:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174641.660080:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174641.660084:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174646.777711:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174646.777717:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174646.779029:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174646.779039:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174646.779399:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174646.779401:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174646.779714:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174646.779716:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174646.779982:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174646.779985:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174651.897075:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174651.897079:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174651.897859:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174651.897866:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174651.898379:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174651.898381:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174651.898669:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174651.898672:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174651.898852:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174651.898854:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174657.017386:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174657.017390:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174657.018368:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174657.018373:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174657.018751:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174657.018753:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174657.018991:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174657.018992:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174657.019206:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174657.019208:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174662.137118:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174662.137124:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174662.137844:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174662.137848:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174662.138052:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174662.138054:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174662.138229:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174662.138230:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174662.138499:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174662.138501:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174667.257265:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174667.257271:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174667.258035:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174667.258037:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174667.258337:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174667.258339:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174667.258579:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174667.258581:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174667.258864:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174667.258867:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174672.377262:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174672.377776:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174672.378277:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174672.378492:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174672.378714:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174677.497888:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174677.497900:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174677.499442:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174677.499446:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174677.499694:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174677.499696:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174677.499906:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174677.499908:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174677.500195:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174677.500197:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174682.617576:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174682.617587:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786174682.619299:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174682.619305:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786174682.619667:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174682.619669:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786174682.620097:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174682.620099:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786174682.620332:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174682.620334:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786174687.738317:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174687.738328:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174687.739896:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174687.739899:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174687.740325:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174687.740328:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174687.740815:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174687.740818:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174687.741208:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174687.741212:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786174692.857720:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174692.857737:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174692.859080:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174692.859084:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174692.859542:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174692.859549:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174692.860035:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174692.860039:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174692.860478:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174692.860481:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786174697.977114:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174697.977119:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174697.977778:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174697.977780:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174697.978081:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174697.978083:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174697.978263:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174697.978265:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174697.978503:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174697.978505:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786174703.097779:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174703.097786:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174703.099119:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174703.099122:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174703.099501:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174703.099505:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174703.099991:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174703.099994:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174703.100434:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174703.100438:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174708.217337:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174708.217342:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174708.217957:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174708.217958:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174708.218191:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174708.218193:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174708.218430:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174708.218432:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174708.218663:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174708.218665:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174713.337657:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174713.337665:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174713.339065:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174713.339066:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174713.339472:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174713.339475:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174713.339749:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174713.339751:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174713.340071:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174713.340073:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174718.457576:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174718.457586:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174718.458456:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174718.458459:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174718.458673:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174718.458675:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174718.458882:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174718.458884:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174718.459084:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174718.459087:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174723.577651:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174723.577658:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1786174723.578891:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174723.578894:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1786174723.579262:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174723.579266:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1786174723.579701:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174723.579721:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1786174723.580236:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174723.580239:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1786174728.697609:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174728.697618:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174728.698916:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174728.698920:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174728.699537:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174728.699539:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174728.699857:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174728.699860:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174728.700050:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174728.700052:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786174733.817787:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174733.817806:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174733.818928:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174733.818931:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174733.819272:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174733.819275:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174733.819539:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174733.819567:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174733.819858:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174733.819860:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786174738.937806:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174738.938755:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174738.939164:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174738.939507:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174738.939886:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174744.058191:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174744.058206:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174744.059286:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174744.059291:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174744.059732:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174744.059736:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174744.060080:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174744.060083:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174744.060379:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174744.060381:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174749.177747:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174749.177755:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174749.178707:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174749.178710:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174749.179163:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174749.179167:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174749.179787:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174749.179792:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174749.180345:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174749.180350:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174754.297988:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174754.298002:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174754.299746:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174754.299749:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174754.300072:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174754.300075:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174754.300406:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174754.300409:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174754.300879:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174754.300882:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174759.417621:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174759.417632:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174759.418735:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174759.418739:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174759.419431:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174759.419436:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174759.420034:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174759.420039:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174759.420646:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174759.420651:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174764.537732:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174764.537742:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174764.538971:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174764.538978:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174764.539773:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174764.539776:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174764.540208:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174764.540213:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786174764.540580:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174764.540583:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174769.657884:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174769.657896:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174769.658794:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174769.658797:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174769.659164:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174769.659169:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174769.659579:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174769.659584:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174769.659854:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174769.659868:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174774.777697:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174774.777707:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174774.779051:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174774.779054:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174774.779467:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174774.779470:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174774.780051:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174774.780055:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174774.780618:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174774.780621:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174779.897873:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174779.897890:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174779.899625:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174779.899641:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174779.900747:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174779.900751:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174779.901392:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174779.901396:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174779.901843:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174779.901847:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174785.017686:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174785.017695:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174785.018799:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174785.018802:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174785.019167:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174785.019172:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174785.019644:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174785.019647:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174785.020022:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174785.020024:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174790.137841:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174790.137848:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174790.139086:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174790.139090:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174790.139613:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174790.139617:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174790.139993:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174790.139996:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174790.140509:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174790.140562:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174795.257298:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174795.257305:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174795.258235:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174795.258238:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174795.258565:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174795.258568:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174795.258850:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174795.258853:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174795.259319:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174795.259323:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174800.377308:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174800.377314:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174800.378054:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174800.378056:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174800.378282:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174800.378284:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174800.378471:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174800.378473:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174800.378704:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174800.378706:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786174805.497625:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174805.498465:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174805.499079:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174805.499440:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174805.499772:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174810.617814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174810.617825:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174810.619007:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174810.619009:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174810.619253:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174810.619258:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174810.619486:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174810.619489:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174810.619936:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174810.619938:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174815.737492:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174815.737499:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174815.738354:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174815.738357:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174815.738767:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174815.738770:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174815.739187:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174815.739191:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174815.739555:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174815.739559:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174820.857303:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174820.857308:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174820.858089:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174820.858092:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174820.858410:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174820.858412:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174820.858750:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174820.858753:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174820.859042:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174820.859045:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174825.977893:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174825.977915:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174825.979562:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174825.979575:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174825.980122:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174825.980125:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174825.980486:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174825.980489:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174825.981060:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174825.981062:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174831.097894:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174831.097909:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174831.098894:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174831.098897:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174831.099219:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174831.099222:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174831.099616:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174831.099619:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174831.099959:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174831.099962:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174836.217255:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174836.217264:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174836.218014:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174836.218029:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174836.218256:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174836.218258:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174836.218488:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174836.218490:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174836.218784:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174836.218786:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174841.337888:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174841.337895:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174841.339448:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174841.339450:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174841.339798:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174841.339800:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174841.340007:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174841.340009:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174841.340351:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174841.340354:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786174846.457903:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174846.457920:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786174846.459312:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174846.459316:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786174846.459833:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174846.459836:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786174846.460260:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174846.460263:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786174846.460684:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174846.460688:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786174851.577380:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174851.577387:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174851.578313:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174851.578318:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174851.578650:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174851.578652:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174851.579104:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174851.579106:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174851.579403:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174851.579406:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786174856.697975:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174856.697990:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174856.699884:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174856.699888:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174856.700263:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174856.700267:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174856.700678:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174856.700681:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174856.701168:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174856.701172:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786174861.817748:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174861.817760:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174861.819096:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174861.819099:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174861.819606:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174861.819609:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174861.820049:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174861.820052:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174861.820668:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174861.820670:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174866.937605:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174866.937614:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174866.938627:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174866.938632:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174866.938982:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174866.938985:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174866.939312:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174866.939315:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174866.939662:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174866.939665:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174872.057552:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174872.058129:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174872.058710:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174872.059052:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174872.059306:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174877.177237:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174877.177245:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174877.178114:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174877.178117:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174877.178421:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174877.178422:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174877.178652:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174877.178654:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174877.178832:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174877.178834:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786174882.297135:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174882.297161:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174882.297808:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174882.297811:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174882.297998:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174882.297999:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174882.298190:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174882.298192:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174882.298372:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174882.298374:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786174887.417580:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174887.417588:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174887.418905:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174887.418909:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174887.419342:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174887.419345:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174887.419697:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174887.419701:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174887.420174:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174887.420177:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786174892.537587:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174892.537593:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174892.538568:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174892.538572:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174892.538921:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174892.538923:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174892.539316:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174892.539321:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174892.539750:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174892.539753:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786174897.657874:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174897.657891:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174897.659232:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174897.659236:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174897.659613:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174897.659616:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174897.659957:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174897.659960:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174897.660367:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174897.660370:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786174902.777590:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174902.777598:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174902.778985:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174902.778988:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174902.779362:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174902.779366:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174902.779744:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174902.779748:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786174902.780256:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174902.780259:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786174907.897396:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174907.897402:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174907.898213:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174907.898217:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174907.898563:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174907.898566:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174907.898985:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174907.898989:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174907.899629:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174907.899632:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786174913.017597:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174913.017610:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174913.019042:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174913.019046:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174913.019398:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174913.019401:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174913.019773:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174913.019776:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174913.020285:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174913.020288:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174918.137755:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174918.137767:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174918.138982:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174918.138985:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174918.139433:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174918.139436:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174918.139863:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174918.139866:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174918.140230:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174918.140233:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174923.257720:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174923.257733:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174923.258993:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174923.258998:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174923.259432:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174923.259435:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174923.259856:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174923.259860:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174923.260310:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174923.260313:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174928.377332:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174928.377337:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174928.378344:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174928.378346:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174928.378668:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174928.378670:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174928.378984:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174928.378987:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174928.379331:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174928.379334:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786174933.497391:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174933.497399:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174933.498087:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174933.498089:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174933.498356:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174933.498358:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174933.498638:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174933.498640:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174933.498924:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174933.498926:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786174938.617809:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174938.618585:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174938.619467:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174938.619879:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786174938.620218:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174943.737642:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174943.737651:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174943.738914:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174943.738917:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174943.739388:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174943.739391:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174943.739742:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174943.739744:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174943.740082:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174943.740084:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786174948.857737:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174948.857745:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174948.858949:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174948.858952:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174948.859327:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174948.859330:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174948.859758:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174948.859762:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174948.860169:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174948.860173:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786174953.977862:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174953.977876:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174953.979041:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174953.979044:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174953.979571:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174953.979575:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174953.980015:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174953.980018:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174953.980502:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174953.980506:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786174959.097494:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174959.097504:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174959.099090:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174959.099102:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174959.099469:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174959.099471:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174959.099839:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174959.099843:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174959.100136:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174959.100139:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786174964.217555:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174964.217565:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174964.218862:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174964.218866:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174964.219237:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174964.219240:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174964.219709:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174964.219713:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174964.220085:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174964.220088:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786174969.337584:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174969.337593:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174969.338874:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174969.338881:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174969.339239:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174969.339242:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174969.339656:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174969.339659:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174969.340165:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174969.340168:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786174974.457212:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174974.457218:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174974.457890:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174974.457891:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174974.458099:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174974.458101:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174974.458331:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174974.458333:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174974.458566:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174974.458568:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1786174979.577671:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174979.577683:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174979.578980:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174979.578983:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174979.579421:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174979.579427:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174979.579895:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174979.579900:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174979.580580:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174979.580584:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786174984.697449:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174984.697455:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174984.698317:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174984.698320:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174984.698726:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174984.698730:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174984.699071:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174984.699074:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174984.699606:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174984.699610:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786174989.817757:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174989.817765:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174989.819013:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174989.819017:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174989.819481:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174989.819484:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174989.819927:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174989.819931:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174989.820504:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174989.820507:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786174994.937959:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174994.937974:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174994.940038:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174994.940077:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174994.940542:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174994.940545:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174994.941007:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174994.941010:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786174994.941609:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786174994.941612:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786175000.057432:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175000.057440:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786175000.058732:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175000.058734:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786175000.059158:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175000.059161:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786175000.059490:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175000.059492:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786175000.060075:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175000.060078:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786175005.177787:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175005.178647:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175005.179077:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175005.179415:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175005.179758:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175010.297814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175010.297826:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175010.299666:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175010.299670:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175010.300160:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175010.300164:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175010.300599:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175010.300603:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175010.301223:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175010.301227:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175015.417592:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175015.417603:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786175015.419046:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175015.419050:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786175015.419582:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175015.419585:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786175015.420052:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175015.420058:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786175015.420726:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175015.420729:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786175020.537309:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175020.537340:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786175020.538112:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175020.538114:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786175020.538522:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175020.538524:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786175020.538777:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175020.538780:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786175020.539238:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175020.539243:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786175025.657156:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175025.657163:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786175025.657857:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175025.657860:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786175025.658104:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175025.658106:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786175025.658290:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175025.658292:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786175025.658553:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175025.658555:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786175030.777456:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175030.777466:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786175030.778240:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175030.778242:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786175030.778654:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175030.778659:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786175030.778979:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175030.778994:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786175030.779336:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175030.779339:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175035.897713:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175035.897780:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175035.899080:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175035.899091:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175035.899466:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175035.899468:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175035.899897:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175035.899900:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175035.900373:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175035.900376:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175041.017682:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175041.017696:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175041.018947:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175041.018950:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175041.019358:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175041.019362:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175041.019817:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175041.019820:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175041.020342:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175041.020347:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175046.137710:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175046.137734:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175046.139422:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175046.139432:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175046.140070:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175046.140071:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175046.140379:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175046.140381:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175046.140715:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175046.140718:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175051.257768:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175051.257778:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175051.259013:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175051.259016:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175051.259382:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175051.259387:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175051.259814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175051.259819:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175051.260380:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175051.260384:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175056.377936:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175056.377947:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175056.379016:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175056.379020:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175056.379374:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175056.379377:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175056.379900:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175056.379904:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175056.380269:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175056.380273:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175061.497368:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175061.497373:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175061.498444:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175061.498449:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175061.499301:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175061.499303:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175061.499679:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175061.499682:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175061.500090:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175061.500093:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175066.617790:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175066.617799:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786175066.619137:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175066.619140:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786175066.619595:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175066.619598:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786175066.619963:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175066.619969:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786175066.620859:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175066.620864:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786175071.737242:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175071.737740:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175071.737921:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175071.738085:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175071.738369:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175076.858281:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175076.858294:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175076.859464:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175076.859468:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175076.860046:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175076.860050:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175076.860659:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175076.860662:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175076.861071:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175076.861075:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175081.977665:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175081.977674:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175081.978442:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175081.978445:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175081.978951:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175081.978954:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175081.980203:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175081.980239:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175081.980747:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175081.980750:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175087.097387:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175087.097396:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175087.098260:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175087.098262:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175087.098564:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175087.098566:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175087.098830:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175087.098832:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175087.099280:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175087.099282:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175092.217399:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175092.217404:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175092.218196:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175092.218199:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175092.218399:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175092.218401:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175092.218637:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175092.218639:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175092.218989:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175092.218990:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175097.337285:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175097.337294:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175097.338018:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175097.338020:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175097.338298:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175097.338300:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175097.338553:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175097.338556:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175097.338797:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175097.338799:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175102.457822:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175102.457832:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175102.458966:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175102.458969:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175102.459362:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175102.459365:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175102.459863:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175102.459868:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175102.460254:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175102.460257:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175107.577561:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175107.577566:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175107.578775:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175107.578782:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175107.579421:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175107.579424:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175107.579911:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175107.579914:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175107.580284:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175107.580288:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175112.697375:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175112.697380:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175112.698385:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175112.698389:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175112.698803:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175112.698805:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175112.699180:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175112.699183:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175112.699469:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175112.699471:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175117.817427:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175117.817433:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175117.818466:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175117.818469:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175117.818817:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175117.818820:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175117.819232:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175117.819235:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175117.819638:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175117.819640:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175122.937776:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175122.937789:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175122.939086:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175122.939092:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175122.939630:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175122.939633:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175122.940090:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175122.940095:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175122.940588:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175122.940592:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175128.057834:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175128.057849:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175128.059051:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175128.059055:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175128.059561:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175128.059565:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175128.060043:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175128.060046:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175128.060499:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175128.060502:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175133.177258:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175133.177263:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175133.177839:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175133.177841:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175133.178087:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175133.178089:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175133.178383:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175133.178385:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175133.178568:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175133.178570:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175138.297360:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175138.297894:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175138.298195:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175138.298765:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175138.299077:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175143.417561:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175143.417567:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175143.418154:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175143.418156:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175143.418436:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175143.418439:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175143.418698:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175143.418700:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175143.418878:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175143.418880:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175148.537641:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175148.537653:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175148.538553:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175148.538554:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175148.538762:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175148.538763:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175148.538942:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175148.538944:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175148.539179:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175148.539181:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175153.657226:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175153.657232:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175153.658018:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175153.658024:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175153.658258:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175153.658261:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175153.658478:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175153.658480:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175153.658814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175153.658817:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175158.777150:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175158.777156:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175158.777801:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175158.777802:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175158.777985:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175158.777987:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175158.778198:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175158.778200:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175158.778443:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175158.778444:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175163.897462:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175163.897469:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175163.898588:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175163.898592:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175163.898938:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175163.898941:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175163.899236:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175163.899239:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175163.899672:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175163.899675:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175169.017872:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175169.017884:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175169.019357:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175169.019390:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175169.019882:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175169.019885:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175169.020274:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175169.020277:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175169.020964:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175169.020968:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175174.138037:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175174.138045:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175174.139274:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175174.139277:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175174.139790:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175174.139792:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175174.140117:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175174.140120:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175174.140498:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175174.140502:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175179.257719:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175179.257734:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175179.258940:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175179.258943:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175179.259299:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175179.259303:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175179.259758:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175179.259762:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175179.260093:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175179.260096:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175184.377928:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175184.377940:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175184.378778:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175184.378782:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175184.379211:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175184.379238:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175184.379654:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175184.379657:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175184.380100:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175184.380104:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175189.497096:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175189.497100:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175189.497617:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175189.497619:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175189.497910:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175189.497912:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175189.498117:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175189.498119:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175189.498380:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175189.498382:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786175194.617847:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175194.617865:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786175194.619562:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175194.619565:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786175194.619956:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175194.619959:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786175194.620250:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175194.620252:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786175194.620788:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175194.620791:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786175199.737832:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175199.737844:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786175199.739344:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175199.739347:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786175199.739711:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175199.739715:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786175199.740123:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175199.740126:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786175199.740437:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175199.740440:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786175204.857548:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175204.858253:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175204.858992:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175204.859422:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175204.859749:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175209.977743:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175209.977750:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175209.979585:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175209.979597:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175209.980160:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175209.980163:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175209.980515:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175209.980518:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175209.981111:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175209.981116:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175215.097853:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175215.097869:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175215.099548:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175215.099556:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175215.100149:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175215.100152:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175215.100494:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175215.100497:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175215.100786:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175215.100789:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175220.217730:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175220.217741:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175220.219060:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175220.219063:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175220.219466:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175220.219478:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175220.219860:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175220.219863:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175220.220392:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175220.220395:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175225.337458:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175225.337465:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175225.338384:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175225.338387:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175225.338795:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175225.338799:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175225.339222:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175225.339226:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175225.339597:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175225.339601:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175230.457758:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175230.457767:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175230.458828:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175230.458831:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175230.459188:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175230.459191:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175230.459553:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175230.459556:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175230.460021:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175230.460025:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175235.578061:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175235.578071:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175235.579710:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175235.579713:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175235.580072:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175235.580074:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175235.580375:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175235.580378:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175235.580851:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175235.580856:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175240.697763:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175240.697777:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175240.699026:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175240.699029:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175240.699398:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175240.699402:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175240.699916:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175240.699919:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175240.700399:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175240.700402:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175245.817938:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175245.817957:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175245.820000:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175245.820014:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175245.820464:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175245.820469:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175245.820880:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175245.820883:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175245.821355:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175245.821358:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175250.937920:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175250.937928:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175250.939053:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175250.939059:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175250.939541:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175250.939545:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175250.939943:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175250.939946:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175250.940394:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175250.940397:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175256.057752:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175256.057763:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175256.058847:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175256.058851:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175256.059284:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175256.059288:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175256.059708:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175256.059712:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175256.060107:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175256.060111:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786175261.177193:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175261.177197:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786175261.177814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175261.177816:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786175261.178001:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175261.178003:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786175261.178197:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175261.178199:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786175261.178443:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175261.178445:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786175266.297666:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175266.297673:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786175266.298699:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175266.298701:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786175266.299110:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175266.299114:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786175266.299703:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175266.299708:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786175266.300089:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175266.300092:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786175271.417731:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175271.418595:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175271.419000:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175271.419342:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175271.419713:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175276.537644:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175276.537655:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175276.538538:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175276.538540:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175276.538783:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175276.538786:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175276.539011:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175276.539013:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175276.539267:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175276.539269:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175281.658024:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175281.658033:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786175281.659802:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175281.659808:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786175281.660300:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175281.660307:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786175281.660719:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175281.660723:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786175281.661376:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175281.661380:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786175286.777808:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175286.777816:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175286.779158:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175286.779162:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175286.779687:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175286.779690:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175286.780098:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175286.780102:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175286.780757:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175286.780760:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175291.897780:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175291.897791:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175291.899449:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175291.899452:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175291.899782:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175291.899783:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175291.900020:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175291.900021:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175291.900303:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175291.900305:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175297.017829:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175297.017839:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175297.018974:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175297.018977:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175297.019377:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175297.019379:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175297.019766:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175297.019769:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175297.020116:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175297.020119:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175302.137699:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175302.137712:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175302.138484:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175302.138486:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175302.138725:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175302.138727:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175302.138904:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175302.138923:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175302.139285:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175302.139287:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175307.257158:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175307.257166:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175307.257884:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175307.257886:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175307.258090:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175307.258092:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175307.258329:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175307.258330:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175307.258614:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175307.258616:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175312.377399:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175312.377409:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175312.378260:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175312.378262:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175312.378448:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175312.378451:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175312.378679:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175312.378681:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175312.378855:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175312.378857:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175317.497151:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175317.497155:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175317.497787:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175317.497789:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175317.498000:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175317.498001:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175317.498178:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175317.498180:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175317.498454:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175317.498456:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175322.617188:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175322.617198:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786175322.617794:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175322.617796:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786175322.618031:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175322.618033:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786175322.618254:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175322.618256:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786175322.618447:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175322.618450:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786175327.737488:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175327.737498:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1786175327.738633:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175327.738635:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1786175327.738969:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175327.738971:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1786175327.739237:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175327.739239:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1786175327.739475:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175327.739478:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1786175332.857384:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175332.857393:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786175332.858379:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175332.858382:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786175332.858685:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175332.858687:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786175332.859027:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175332.859029:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786175332.859368:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175332.859371:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786175337.977162:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175337.977568:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175337.977765:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175337.977931:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175337.978095:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175343.097719:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175343.097727:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175343.098472:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175343.098474:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175343.098823:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175343.098826:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175343.099091:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175343.099093:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175343.099350:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175343.099352:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175348.217839:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175348.217851:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175348.219057:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175348.219060:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175348.219425:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175348.219428:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175348.219866:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175348.219870:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175348.220375:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175348.220378:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175353.337498:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175353.337506:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175353.338458:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175353.338462:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175353.338988:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175353.338993:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175353.339445:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175353.339448:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175353.339958:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175353.339962:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175358.457594:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175358.457600:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175358.458813:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175358.458816:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175358.459191:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175358.459193:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175358.459493:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175358.459495:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175358.459832:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175358.459835:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175363.577283:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175363.577291:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175363.578120:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175363.578122:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175363.578332:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175363.578333:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175363.578652:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175363.578655:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175363.579074:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175363.579076:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175368.697237:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175368.697242:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175368.697997:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175368.698003:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175368.698415:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175368.698417:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175368.698738:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175368.698740:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175368.698971:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175368.698973:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175373.817228:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175373.817233:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175373.817851:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175373.817853:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175373.818088:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175373.818090:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175373.818272:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175373.818274:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175373.818500:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175373.818503:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175378.937206:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175378.937211:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175378.937989:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175378.937994:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175378.938360:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175378.938364:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175378.938692:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175378.938695:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175378.938887:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175378.938889:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175384.057261:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175384.057268:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175384.058081:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175384.058083:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175384.058377:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175384.058380:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175384.058680:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175384.058682:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175384.058990:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175384.058993:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175389.177336:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175389.177342:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175389.178049:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175389.178051:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175389.178280:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175389.178282:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175389.178466:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175389.178477:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175389.178762:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175389.178764:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175394.297392:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175394.297401:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175394.298649:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175394.298653:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175394.299060:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175394.299062:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175394.299439:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175394.299445:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175394.300013:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175394.300016:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175399.417182:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175399.417186:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175399.417781:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175399.417783:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175399.417978:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175399.417981:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175399.418277:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175399.418278:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175399.418463:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175399.418465:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175404.537323:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175404.537808:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175404.538081:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175404.538265:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175404.538977:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175409.657600:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175409.657607:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175409.658342:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175409.658344:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175409.658703:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175409.658706:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175409.659030:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175409.659033:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175409.659368:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175409.659371:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175414.777288:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175414.777296:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175414.777858:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175414.777860:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175414.778095:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175414.778096:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175414.778322:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175414.778324:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175414.778551:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175414.778553:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175419.897800:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175419.897812:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175419.899168:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175419.899172:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175419.899635:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175419.899638:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175419.900029:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175419.900032:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175419.900615:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175419.900619:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175425.017228:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175425.017234:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175425.018052:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175425.018054:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175425.018252:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175425.018254:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175425.018430:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175425.018432:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175425.018732:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175425.018734:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175430.137688:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175430.137697:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175430.139025:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175430.139034:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175430.139701:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175430.139704:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175430.140224:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175430.140227:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175430.140568:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175430.140571:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175435.257471:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175435.257479:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175435.258160:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175435.258164:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175435.258454:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175435.258458:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175435.258778:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175435.258780:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175435.259072:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175435.259076:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175440.377576:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175440.377591:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786175440.378656:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175440.378657:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786175440.378916:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175440.378919:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786175440.379132:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175440.379133:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786175440.379357:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175440.379359:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786175445.497274:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175445.497282:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786175445.497876:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175445.497879:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786175445.498196:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175445.498199:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786175445.498498:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175445.498501:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786175445.498835:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175445.498838:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786175450.617139:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175450.617143:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175450.617789:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175450.617791:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175450.617979:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175450.617981:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175450.618157:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175450.618159:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175450.618361:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175450.618363:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175455.737862:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175455.737871:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175455.739100:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175455.739106:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175455.739546:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175455.739550:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175455.740118:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175455.740121:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175455.740501:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175455.740504:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175460.858052:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175460.858061:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175460.859029:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175460.859031:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175460.859309:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175460.859313:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175460.859715:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175460.859726:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175460.860135:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175460.860138:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175465.977924:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175465.977944:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175465.979625:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175465.979642:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175465.979969:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175465.979971:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175465.980244:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175465.980247:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175465.980677:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175465.980679:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175471.097331:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175471.097981:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175471.098427:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175471.098648:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175471.098805:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175476.217542:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175476.217548:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175476.218568:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175476.218572:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175476.218950:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175476.218953:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175476.219292:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175476.219296:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175476.219705:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175476.219708:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175481.337666:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175481.337694:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175481.338831:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175481.338838:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175481.339434:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175481.339436:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175481.339969:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175481.339983:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175481.340310:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175481.340316:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175486.457839:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175486.457851:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175486.459049:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175486.459053:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175486.459535:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175486.459540:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175486.459992:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175486.459996:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175486.460445:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175486.460449:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175491.577431:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175491.577445:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175491.578748:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175491.578750:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175491.579018:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175491.579021:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175491.579277:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175491.579280:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175491.579720:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175491.579722:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175496.697117:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175496.697123:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175496.697752:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175496.697754:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175496.697944:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175496.697945:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175496.698119:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175496.698121:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175496.698341:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175496.698343:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175501.817190:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175501.817197:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175501.817784:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175501.817786:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175501.817976:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175501.817978:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175501.818168:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175501.818170:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175501.818352:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175501.818354:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175506.937429:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175506.937437:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175506.938468:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175506.938470:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175506.938824:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175506.938828:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175506.939138:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175506.939142:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175506.939628:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175506.939647:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175512.057252:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175512.057257:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175512.058147:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175512.058149:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175512.058367:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175512.058369:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175512.058739:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175512.058741:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175512.059109:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175512.059111:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175517.177274:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175517.177279:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175517.177980:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175517.177982:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175517.178354:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175517.178356:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175517.178630:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175517.178632:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175517.179085:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175517.179087:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175522.297166:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175522.297172:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175522.297836:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175522.297838:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175522.298037:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175522.298038:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175522.298259:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175522.298261:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175522.298507:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175522.298523:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175527.417473:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175527.417480:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175527.418238:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175527.418240:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175527.418559:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175527.418561:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175527.418813:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175527.418815:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175527.419097:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175527.419099:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175532.537638:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175532.537649:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175532.538671:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175532.538675:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175532.539187:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175532.539191:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175532.539754:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175532.539758:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175532.540256:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175532.540260:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175537.657185:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175537.657702:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175537.657938:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175537.658118:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175537.658353:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175542.777295:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175542.777301:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175542.778316:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175542.778320:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175542.778769:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175542.778775:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175542.779101:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175542.779103:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175542.779363:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175542.779365:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175547.897228:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175547.897232:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175547.898149:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175547.898151:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175547.898526:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175547.898529:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175547.899025:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175547.899028:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175547.899517:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175547.899520:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175553.017344:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175553.017350:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175553.018120:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175553.018125:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175553.018326:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175553.018328:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175553.018503:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175553.018505:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175553.018846:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175553.018848:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786175558.137688:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175558.137698:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175558.138270:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175558.138272:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175558.138521:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175558.138524:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175558.138859:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175558.138861:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175558.139134:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175558.139136:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786175563.257217:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175563.257223:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175563.257935:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175563.257939:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175563.258198:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175563.258200:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175563.258395:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175563.258397:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175563.258709:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175563.258710:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786175568.378130:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175568.378149:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175568.379425:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175568.379439:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175568.380122:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175568.380135:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175568.380740:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175568.380745:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175568.381400:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175568.381404:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786175573.497144:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175573.497147:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786175573.497817:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175573.497819:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786175573.498113:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175573.498116:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786175573.498448:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175573.498451:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786175573.498787:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175573.498790:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786175578.617433:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175578.617441:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175578.618276:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175578.618279:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175578.618648:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175578.618651:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175578.618967:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175578.618971:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175578.619368:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175578.619371:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175583.737711:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175583.737724:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175583.738837:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175583.738840:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175583.739231:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175583.739235:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175583.739691:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175583.739694:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175583.740112:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175583.740115:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175588.857761:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175588.857772:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175588.858970:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175588.858974:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175588.859623:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175588.859628:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175588.860201:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175588.860206:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175588.860652:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175588.860655:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175593.977413:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175593.977420:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175593.978493:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175593.978498:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175593.978895:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175593.978897:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175593.979245:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175593.979247:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175593.979535:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175593.979538:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175599.097769:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175599.097781:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175599.099289:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175599.099293:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175599.099710:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175599.099714:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175599.100301:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175599.100305:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175599.100893:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175599.100898:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175604.217813:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175604.218783:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175604.219019:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175604.219259:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175604.219445:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175609.339028:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175609.339035:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175609.339678:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175609.339681:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175609.340206:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175609.340209:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175609.340600:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175609.340603:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786175609.341074:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175609.341078:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00080000:2.0:1786175614.456673:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175614.456675:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175614.456677:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175614.456680:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175614.456682:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.456683:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175614.456685:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175614.456693:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786175614.456696:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175614.456699:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175614.456701:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.456703:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175614.456706:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175614.456709:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175614.456717:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786175614.456722:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175614.456731:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856150912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175614.456741:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175614.456746:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175614.456751:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856150912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175614.456775:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175614.456779:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175614.456782:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175614.456783:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175614.456784:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175614.456786:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175614.456788:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5837
00000400:00000200:2.0:1786175614.456791:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175614.456794:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175614.456796:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175614.456798:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.456801:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175614.456803:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.456805:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175614.456807:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175614.456808:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175614.456809:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175614.456811:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175614.456812:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175614.456814:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175614.456819:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.456823:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175614.456827:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.456829:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175614.456832:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175614.456836:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175614.456841:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175614.456844:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175614.456851:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175614.456875:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.456879:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175614.456884:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175614.456889:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.456892:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175614.456898:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786175614.456903:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175614.456908:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175614.456910:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.456915:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175614.456917:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175614.456923:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175614.456932:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786175614.456939:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175614.456951:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856151040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175614.456962:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175614.456966:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175614.456971:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856151040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175614.456982:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175614.456987:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175614.456991:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175614.456994:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175614.456997:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175614.457001:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175614.457006:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5837
00000400:00000200:2.0:1786175614.457021:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175614.457029:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175614.457032:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175614.457038:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457042:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175614.457047:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457052:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175614.457054:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175614.457056:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175614.457080:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175614.457083:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000800:00000200:3.1:1786175614.457087:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786175614.457088:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175614.457093:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:3.1:1786175614.457093:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:2.0:1786175614.457101:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457105:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175614.457111:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457114:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175614.457118:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175614.457124:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457127:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175614.457129:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175614.457134:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175614.457138:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.457141:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175614.457145:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175614.457149:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.457154:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175614.457160:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786175614.457164:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175614.457170:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175614.457172:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.457176:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175614.457178:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175614.457181:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175614.457186:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000001:2.0:1786175614.457190:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175614.457197:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856151168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175614.457210:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175614.457213:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175614.457220:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856151168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175614.457230:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175614.457233:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175614.457241:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175614.457246:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175614.457249:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175614.457253:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175614.457265:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5837
00000400:00000200:2.0:1786175614.457274:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175614.457280:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175614.457282:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175614.457288:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457291:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175614.457295:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457298:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175614.457302:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175614.457304:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175614.457307:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175614.457310:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175614.457314:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175614.457318:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175614.457324:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457329:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175614.457333:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457336:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175614.457341:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175614.457347:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457353:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175614.457357:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175614.457362:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175614.457369:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.457374:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175614.457379:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175614.457385:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.457391:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175614.457399:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786175614.457407:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175614.457413:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175614.457417:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.457420:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175614.457423:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175614.457426:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175614.457431:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786175614.457436:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175614.457443:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856151296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175614.457463:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175614.457467:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175614.457473:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856151296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175614.457485:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175614.457488:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175614.457490:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175614.457491:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175614.457492:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175614.457496:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175614.457501:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5837
00000400:00000200:2.0:1786175614.457507:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175614.457538:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175614.457542:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175614.457546:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457548:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175614.457558:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457562:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175614.457571:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175614.457573:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175614.457577:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175614.457580:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175614.457583:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175614.457588:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175614.457593:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457596:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175614.457599:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457603:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175614.457609:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175614.457615:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175614.457620:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175614.457625:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175614.457629:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175614.457633:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.457638:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175614.457642:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175614.457648:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.457651:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175614.457655:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786175614.457659:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175614.457661:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175614.457665:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175614.457667:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175614.457669:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175614.457673:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175614.457679:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786175614.457683:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175614.457690:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856151424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175614.457709:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175614.457713:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175614.457719:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856151424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175614.457735:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175614.457743:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5847)
00000800:00000200:3.1:1786175614.457780:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175614.457782:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175614.457967:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175614.457969:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175614.458176:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175614.458177:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175614.458424:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175614.458425:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:2.0:1786175616.056913:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175616.056921:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175616.056928:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175619.128771:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175619.128777:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175619.128782:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786175619.577694:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175619.577702:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175619.578926:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175619.578938:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175619.579471:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175619.579474:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175619.579955:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175619.579959:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175619.580403:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175619.580406:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000020:00000001:2.0:1786175622.200732:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175622.200739:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175622.200744:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786175624.697726:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175624.697734:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175624.699098:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175624.699102:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175624.699469:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175624.699473:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175624.699935:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175624.699941:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175624.700487:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175624.700491:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:2.0:1786175625.272886:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175625.272893:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175625.272898:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786175629.816725:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175629.816739:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175629.816744:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175629.816750:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175629.816757:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175629.816768:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 5853
00000400:00000200:2.0:1786175629.816778:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175629.816787:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175629.816792:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175629.816796:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.816811:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175629.816815:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.816818:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175629.816823:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175629.816826:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175629.816829:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175629.816833:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175629.816837:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175629.816841:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175629.816845:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.816847:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175629.816850:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.816852:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175629.816856:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175629.816860:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175629.816863:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175629.816878:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175629.816882:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175629.816886:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.816888:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175629.816892:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175629.816903:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000020:00000040:2.0:1786175629.816908:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175629.816949:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175629.816951:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.816955:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175629.816958:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175629.816964:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175629.816975:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786175629.816983:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175629.816996:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856152832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175629.817009:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175629.817015:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175629.817022:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856152832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175629.817054:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175629.817061:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175629.817069:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175629.817072:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175629.817075:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175629.817080:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175629.817086:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5853
00000400:00000200:2.0:1786175629.817094:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175629.817102:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175629.817107:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175629.817114:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.817119:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175629.817126:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.817131:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175629.817137:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175629.817141:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175629.817145:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175629.817149:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175629.817154:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175629.817160:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175629.817170:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.817177:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175629.817181:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.817188:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175629.817193:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175629.817201:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175629.817212:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175629.817223:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175629.817240:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175629.817250:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.817272:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175629.817279:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175629.817296:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.817320:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175629.817334:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000020:00000040:2.0:1786175629.817346:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175629.817352:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175629.817359:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.817364:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175629.817373:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175629.817379:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175629.817396:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786175629.817408:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175629.817423:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856152960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175629.817448:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175629.817454:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175629.817467:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856152960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175629.817491:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175629.817497:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175629.817503:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175629.817507:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175629.817564:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175629.817569:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175629.817576:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5853
00000400:00000200:2.0:1786175629.817587:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175629.817594:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175629.817600:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175629.817610:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.817622:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175629.817640:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.817655:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175629.817675:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175629.817686:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175629.817700:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175629.817712:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175629.817724:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175629.817740:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175629.817750:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.817765:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175629.817782:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.817788:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175629.817791:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175629.817795:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175629.817798:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175629.817805:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175629.817816:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175629.817828:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.817842:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175629.817849:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175629.817865:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.817880:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175629.817893:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000020:00000040:2.0:1786175629.817908:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175629.817939:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175629.817942:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.817952:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175629.817959:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175629.817966:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175629.817976:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786175629.817983:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175629.817997:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856153088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175629.818036:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175629.818046:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175629.818060:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856153088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1786175629.818074:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175629.818084:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:2.0:1786175629.818137:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175629.818148:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175629.818168:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175629.818172:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175629.818183:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175629.818203:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175629.818214:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5853
00000400:00000200:2.0:1786175629.818236:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175629.818261:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175629.818274:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175629.818293:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.818310:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175629.818315:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.818322:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175629.818333:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175629.818341:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175629.818352:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175629.818358:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175629.818369:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175629.818374:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175629.818388:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.818401:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175629.818404:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.818408:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175629.818414:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175629.818432:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175629.818439:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175629.818448:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175629.818456:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175629.818467:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.818482:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175629.818487:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175629.818492:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.818495:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175629.818551:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000020:00000040:2.0:1786175629.818557:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175629.818561:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175629.818563:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.818567:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175629.818573:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175629.818578:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175629.818589:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786175629.818607:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175629.818628:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856153216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175629.818725:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175629.818735:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175629.818754:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856153216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175629.818788:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175629.818804:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175629.818826:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175629.818838:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175629.818849:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175629.818875:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175629.818900:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5853
00000400:00000200:2.0:1786175629.818947:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175629.818957:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175629.818972:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175629.818992:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.819001:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175629.819008:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.819017:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175629.819024:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175629.819031:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175629.819035:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175629.819040:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175629.819047:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175629.819062:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175629.819089:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.819112:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175629.819125:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175629.819145:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175629.819152:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175629.819170:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175629.819183:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175629.819190:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175629.819197:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175629.819205:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.819208:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175629.819216:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175629.819222:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.819233:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175629.819246:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786175629.819262:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175629.819270:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175629.819274:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175629.819288:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175629.819293:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175629.819300:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175629.819309:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:2.0:1786175629.819317:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175629.819333:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856153344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175629.819396:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175629.819409:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175629.819419:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856153344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175629.819440:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175629.819460:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5863)
00000800:00000200:3.1:1786175629.819766:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175629.819770:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175629.820228:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175629.820231:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175629.820607:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175629.820611:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175629.820967:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175629.820971:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:2.0:1786175630.840704:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175630.840714:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175630.840717:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175630.840724:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175630.840728:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175630.840732:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175630.840736:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175630.840739:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175630.840742:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175631.864610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175631.864616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175631.864617:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175631.864622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175631.864625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175631.864628:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175631.864630:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175631.864632:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175631.864634:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175632.888563:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175632.888569:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175632.888571:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175632.888576:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175632.888579:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175632.888582:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175632.888584:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175632.888586:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175632.888588:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175633.912635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175633.912646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175633.912649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175633.912657:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175633.912662:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175633.912666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175633.912670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175633.912673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175633.912677:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175634.936635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175634.936646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175634.936649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175634.936657:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175634.936662:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175634.936667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175634.936672:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175634.936676:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175634.936680:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175634.937688:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175634.937697:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175634.939085:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175634.939088:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175634.939463:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175634.939468:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175634.939913:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175634.939917:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175634.940472:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175634.940477:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1786175635.960640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175635.960649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175635.960652:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175635.960659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175635.960664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175635.960668:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175635.960672:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175635.960676:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175635.960679:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175636.984575:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175636.984582:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175636.984584:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175636.984590:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175636.984594:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175636.984597:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175636.984600:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175636.984603:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175636.984606:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175638.008654:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175638.008667:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175638.008682:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175638.008693:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175638.008700:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175638.008707:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175638.008712:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175638.008717:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175638.008722:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175639.032620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175639.032629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175639.032632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175639.032639:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175639.032644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175639.032648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175639.032653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175639.032658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175639.032663:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175640.056638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175640.056652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175640.056661:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175640.056675:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175640.056688:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175640.056698:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175640.056707:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175640.056716:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175640.056728:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175640.057561:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175640.057568:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175640.058355:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175640.058359:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175640.058769:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175640.058773:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175640.059207:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175640.059210:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175640.059692:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175640.059695:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:2.0:1786175641.080593:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175641.080599:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175641.080601:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175641.080605:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175641.080607:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175641.080610:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175641.080613:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175641.080615:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175641.080617:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175642.104645:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175642.104652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175642.104654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175642.104659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175642.104663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175642.104666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175642.104669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175642.104671:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175642.104674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175643.128660:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175643.128684:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175643.128687:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175643.128693:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175643.128697:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175643.128702:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175643.128706:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175643.128710:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175643.128713:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175644.152609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175644.152620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175644.152623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175644.152630:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175644.152635:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175644.152640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175644.152644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175644.152648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175644.152651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175645.176661:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175645.176671:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175645.176674:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175645.176682:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175645.176687:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175645.176694:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175645.176698:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175645.176705:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175645.176714:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786175645.177651:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175645.177660:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786175645.179009:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175645.179018:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786175645.179570:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175645.179572:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786175645.179993:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175645.179996:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786175645.180607:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175645.180610:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1786175646.200608:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175646.200617:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175646.200619:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175646.200625:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175646.200629:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175646.200633:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175646.200635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175646.200638:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175646.200640:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175647.224693:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175647.224707:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175647.224710:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175647.224718:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175647.224724:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175647.224730:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175647.224734:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175647.224739:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175647.224767:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175648.248649:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175648.248663:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175648.248667:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175648.248678:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175648.248684:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175648.248690:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175648.248695:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175648.248700:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175648.248705:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175649.272668:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175649.272681:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175649.272684:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175649.272692:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175649.272698:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175649.272704:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175649.272707:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175649.272711:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175649.272715:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175650.296605:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175650.296620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175650.296623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175650.296627:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175650.296636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175650.296640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175650.296643:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175650.296645:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175650.296648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786175650.297306:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175650.297310:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786175650.297935:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175650.297937:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786175650.298144:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175650.298145:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786175650.298360:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175650.298362:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786175650.298559:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175650.298561:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1786175651.320731:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175651.320750:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175651.320752:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175651.320759:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175651.320763:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175651.320767:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175651.320770:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175651.320781:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175651.320785:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175652.344642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175652.344651:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175652.344654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175652.344660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175652.344666:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175652.344670:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175652.344673:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175652.344676:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175652.344679:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175653.368711:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175653.368724:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175653.368728:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175653.368737:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175653.368744:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175653.368751:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175653.368765:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175653.368770:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175653.368774:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175654.392615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175654.392624:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175654.392627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175654.392632:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175654.392636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175654.392640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175654.392644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175654.392647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175654.392649:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175655.416610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175655.416617:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175655.416619:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175655.416624:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175655.416629:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175655.416638:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175655.416641:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175655.416644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175655.416646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786175655.417405:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175655.417415:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786175655.418310:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175655.418313:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786175655.418637:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175655.418639:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786175655.418936:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175655.418938:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786175655.419238:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175655.419242:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1786175656.440802:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175656.440833:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175656.440836:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175656.440844:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175656.440849:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175656.440854:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175656.440872:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175656.440878:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175656.440883:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175657.464626:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175657.464636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175657.464639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175657.464700:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175657.464703:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175657.464706:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175657.464709:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175657.464711:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175657.464714:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786175660.537858:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175660.537865:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786175660.539173:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175660.539180:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786175660.539791:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175660.539795:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786175660.540250:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175660.540253:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786175660.540690:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175660.540693:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786175665.657115:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175665.657118:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786175665.657802:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175665.657804:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786175665.658005:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175665.658006:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786175665.658196:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175665.658199:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786175665.658460:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175665.658462:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786175670.777862:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175670.778470:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175670.779062:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175670.779342:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175670.779636:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000400:00000100:2.0:1786175673.849075:0:2379:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:2.0:1786175673.849127:0:2379:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1786175673.849141:0:2379:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175673.849148:0:2379:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:2.0:1786175673.849157:0:2379:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175673.849165:0:2379:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1786175673.849173:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786175673.849180:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:2.0:1786175673.849190:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786175673.849198:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786175673.849208:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cd48
00000400:00000010:2.0:1786175673.849213:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cd48.
00000100:00000001:2.0:1786175673.849221:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786175673.849236:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba41aa00 x1872934856157952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786175695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175673.849276:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786175673.849285:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856157952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786175695 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175673.849301:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175673.849306:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786175673.849324:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786175673.849330:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881aa00.
00000400:00000200:2.0:1786175673.849338:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786175673.849353:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786175673.849364:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:2.0:1786175673.849373:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:2.0:1786175673.849383:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786175673.849400:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba41b100 x1872934856158080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175673.849437:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786175673.849452:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856158080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175695 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175673.849480:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175673.849642:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786175673.849667:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786175673.849678:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a500.
00000400:00000200:2.0:1786175673.849692:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786175673.849713:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786175673.849726:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c330
00000400:00000010:2.0:1786175673.849736:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c330.
00000100:00000001:2.0:1786175673.849750:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786175673.849769:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba419180 x1872934856158208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175673.849811:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786175673.849829:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856158208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175695 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175673.849868:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175673.849878:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786175673.849905:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786175673.849916:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a100.
00000400:00000200:2.0:1786175673.849923:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786175673.849944:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786175673.849961:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c088
00000400:00000010:2.0:1786175673.849971:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c088.
00000100:00000001:2.0:1786175673.849984:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786175673.849999:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba41ad80 x1872934856158336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175673.850033:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786175673.850047:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856158336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175695 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175673.850084:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175673.850094:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786175673.850120:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786175673.850134:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:2.0:1786175673.850147:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786175673.850167:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786175673.851025:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c5d8
00000400:00000010:2.0:1786175673.851035:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c5d8.
00000100:00000001:2.0:1786175673.851044:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786175673.851059:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba41b480 x1872934856158464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175673.851097:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786175673.851113:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856158464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175695 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175673.851141:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175673.851151:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000200:1.1:1786175675.897277:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175675.897284:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175675.898192:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175675.898194:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175675.898421:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175675.898423:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175675.898643:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175675.898645:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786175675.898930:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175675.898931:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00080000:2.0:1786175681.016682:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175681.016694:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175681.016697:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175681.016703:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175681.016708:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175681.016717:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 5904
00000400:00000200:2.0:1786175681.016730:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175681.016742:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175681.016749:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175681.016755:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.016766:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175681.016770:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.016776:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175681.016781:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175681.016788:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175681.016792:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175681.016799:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175681.016803:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175681.016811:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175681.016817:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.016823:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175681.016828:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.016840:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175681.016845:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175681.016850:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175681.016854:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175681.016859:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175681.016863:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175681.016867:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.016869:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175681.016872:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175681.016883:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786175681.016887:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175681.016892:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175681.016894:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.016898:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175681.016901:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175681.016907:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175681.016915:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786175681.016923:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175681.016934:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856159232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175681.016947:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175681.016952:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175681.016958:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856159232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175681.016986:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175681.016990:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175681.016994:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175681.016996:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175681.016999:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175681.017002:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175681.017005:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5904
00000400:00000200:2.0:1786175681.017010:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175681.017014:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175681.017017:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175681.017021:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017025:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175681.017029:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017032:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175681.017035:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175681.017037:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175681.017038:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175681.017041:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175681.017044:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175681.017053:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175681.017058:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017065:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175681.017071:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017075:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175681.017080:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175681.017086:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017095:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175681.017100:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175681.017108:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175681.017115:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.017122:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175681.017132:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175681.017141:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.017144:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175681.017153:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786175681.017160:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175681.017184:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175681.017189:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.017194:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175681.017197:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175681.017203:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175681.017213:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786175681.017222:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175681.017246:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856159360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175681.017268:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175681.017273:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175681.017280:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856159360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175681.017299:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175681.017306:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175681.017315:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175681.017317:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175681.017328:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175681.017335:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175681.017340:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5904
00000400:00000200:2.0:1786175681.017352:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175681.017358:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175681.017365:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175681.017371:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017375:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175681.017380:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017388:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175681.017396:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175681.017401:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175681.017407:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175681.017411:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175681.017436:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175681.017441:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175681.017449:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017473:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175681.017480:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017486:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175681.017493:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175681.017500:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017504:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175681.017506:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175681.017510:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175681.017512:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.017514:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175681.017516:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175681.017519:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.017521:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175681.017592:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786175681.017598:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175681.017609:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175681.017612:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.017616:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175681.017621:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175681.017627:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175681.017635:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000001:2.0:1786175681.017644:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175681.017663:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856159488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175681.017683:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175681.017689:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175681.017698:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856159488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175681.017715:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175681.017723:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175681.017732:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175681.017739:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175681.017742:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175681.017750:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175681.017757:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5904
00000400:00000200:2.0:1786175681.017768:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175681.017778:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175681.017785:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175681.017788:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017792:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175681.017799:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017804:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175681.017809:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175681.017813:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175681.017819:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175681.017826:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175681.017834:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175681.017838:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175681.017848:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017855:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175681.017859:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017862:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175681.017866:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175681.017874:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175681.017878:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175681.017881:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175681.017887:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175681.017891:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.017898:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175681.017902:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175681.017912:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.017916:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175681.017923:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000800:00000200:1.1:1786175681.017923:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1786175681.017931:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175681.017936:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:1.1:1786175681.017938:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
02000000:00000001:2.0:1786175681.017939:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.017944:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175681.017948:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175681.017954:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175681.017960:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786175681.017965:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175681.017977:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856159616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175681.017988:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175681.017993:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175681.018004:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856159616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175681.018026:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175681.018031:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175681.018039:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175681.018041:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175681.018046:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175681.018052:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175681.018057:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5904
00000400:00000200:2.0:1786175681.018065:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175681.018071:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175681.018075:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175681.018082:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.018087:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175681.018092:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.018099:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175681.018105:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175681.018108:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175681.018113:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175681.018120:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175681.018123:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175681.018126:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175681.018134:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.018139:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175681.018144:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175681.018151:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175681.018157:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175681.018166:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175681.018173:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175681.018177:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175681.018182:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175681.018188:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.018194:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175681.018199:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175681.018206:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.018212:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175681.018216:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786175681.018223:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175681.018236:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175681.018238:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175681.018240:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175681.018243:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175681.018246:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175681.018250:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786175681.018257:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175681.018287:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856159744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175681.018315:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175681.018327:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175681.018335:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856159744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175681.018354:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175681.018361:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5914)
00000800:00000200:1.1:1786175681.019066:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175681.019072:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786175681.019768:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175681.019774:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786175681.020397:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175681.020403:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786175681.021105:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175681.021121:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786175686.137318:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175686.137323:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786175686.138032:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175686.138035:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786175686.138479:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175686.138481:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786175686.138804:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175686.138807:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786175686.139243:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175686.139247:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1786175687.160642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175687.160649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175687.160651:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175687.160655:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175687.160659:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175687.160662:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175687.160664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175687.160666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175687.160669:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175688.184629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175688.184638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175688.184652:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175688.184659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175688.184664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175688.184669:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175688.184672:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175688.184674:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175688.184677:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175689.208627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175689.208638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175689.208642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175689.208664:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175689.208669:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175689.208673:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175689.208677:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175689.208681:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175689.208684:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175690.232636:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175690.232643:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175690.232645:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175690.232650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175690.232653:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175690.232657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175690.232659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175690.232661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175690.232664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175691.256680:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175691.256691:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175691.256702:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175691.256716:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175691.256729:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175691.256740:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175691.256748:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175691.256753:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175691.256761:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786175691.258011:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175691.258028:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786175691.259293:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175691.259296:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786175691.259658:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175691.259660:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786175691.259935:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175691.259937:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786175691.260223:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175691.260227:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1786175692.280621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175692.280631:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175692.280634:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175692.280640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175692.280644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175692.280648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175692.280651:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175692.280654:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175692.280658:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175693.304633:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175693.304642:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175693.304645:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175693.304652:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175693.304657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175693.304661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175693.304665:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175693.304668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175693.304671:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175694.328634:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175694.328643:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175694.328646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175694.328651:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175694.328656:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175694.328661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175694.328665:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175694.328669:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175694.328672:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175695.352648:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175695.352657:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175695.352660:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175695.352666:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175695.352671:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175695.352675:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175695.352679:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175695.352683:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175695.352688:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175696.376659:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175696.376669:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175696.376673:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175696.376680:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175696.376688:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175696.376693:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175696.376698:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175696.376703:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175696.376711:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786175696.377902:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175696.377919:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786175696.378861:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175696.378865:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786175696.379315:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175696.379319:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786175696.379768:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175696.379771:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786175696.380164:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175696.380167:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:2.0:1786175697.400638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175697.400647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175697.400650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175697.400658:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175697.400663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175697.400668:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175697.400672:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175697.400675:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175697.400678:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175698.424620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175698.424630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175698.424633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175698.424640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175698.424645:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175698.424649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175698.424654:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175698.424658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175698.424662:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175699.448621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175699.448632:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175699.448635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175699.448642:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175699.448647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175699.448651:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175699.448654:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175699.448658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175699.448662:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175700.472626:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175700.472636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175700.472639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175700.472646:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175700.472651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175700.472656:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175700.472660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175700.472663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175700.472666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175701.496586:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175701.496595:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175701.496597:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175701.496600:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175701.496604:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175701.496610:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 5924
00000400:00000200:2.0:1786175701.496618:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175701.496624:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175701.496628:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175701.496633:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.496638:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175701.496643:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.496646:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175701.496650:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175701.496653:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175701.496657:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175701.496660:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175701.496663:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175701.496665:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175701.496667:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.496668:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175701.496670:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.496671:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175701.496674:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175701.496677:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175701.496678:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175701.496680:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175701.496683:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175701.496685:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.496686:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175701.496688:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175701.496693:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786175701.496696:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175701.496699:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175701.496701:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.496703:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175701.496705:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175701.496709:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175701.496715:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786175701.496720:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175701.496728:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856161792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175701.496736:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175701.496739:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175701.496743:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856161792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175701.496763:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175701.496765:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175701.496768:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175701.496769:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175701.496770:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175701.496772:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175701.496774:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5924
00000400:00000200:2.0:1786175701.496777:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175701.496780:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175701.496782:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175701.496785:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.496789:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175701.496794:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.496797:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175701.496798:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175701.496801:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175701.496802:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175701.496805:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175701.496810:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175701.496822:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175701.496828:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.496831:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175701.496836:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.496838:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175701.496843:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175701.496848:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175701.496850:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175701.496854:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175701.496859:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175701.496862:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.496865:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175701.496868:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175701.496871:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.496873:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175701.496877:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000020:00000040:2.0:1786175701.496881:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175701.496885:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175701.496887:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.496890:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175701.496893:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175701.496894:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175701.496902:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786175701.496905:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175701.496911:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856161920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175701.496925:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175701.496929:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175701.496938:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856161920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175701.496953:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175701.496958:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175701.496964:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175701.496966:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175701.496968:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175701.496972:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175701.496975:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5924
00000400:00000200:2.0:1786175701.496982:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175701.496986:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175701.496990:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175701.496998:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497001:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175701.497005:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497008:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175701.497011:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175701.497013:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175701.497016:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175701.497019:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175701.497023:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175701.497027:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175701.497033:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497036:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175701.497039:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497044:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175701.497049:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175701.497057:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497062:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175701.497067:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175701.497073:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175701.497078:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.497083:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000800:00000200:1.1:1786175701.497083:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000008:00100000:2.0:1786175701.497087:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000800:00000200:1.1:1786175701.497088:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000008:00000001:2.0:1786175701.497096:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.497101:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175701.497106:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000020:00000040:2.0:1786175701.497112:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175701.497127:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175701.497130:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.497134:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175701.497136:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175701.497139:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175701.497145:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786175701.497173:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175701.497189:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856162048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175701.497209:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175701.497242:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175701.497266:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856162048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175701.497279:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175701.497282:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175701.497291:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175701.497294:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175701.497296:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175701.497300:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175701.497305:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5924
00000400:00000200:2.0:1786175701.497310:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175701.497315:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175701.497316:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175701.497318:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497320:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175701.497323:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497328:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175701.497332:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175701.497335:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175701.497339:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175701.497342:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175701.497346:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175701.497350:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175701.497354:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497357:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175701.497360:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497361:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175701.497363:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175701.497369:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497375:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175701.497380:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175701.497383:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175701.497387:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.497389:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175701.497395:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175701.497399:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.497402:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175701.497408:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786175701.497412:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175701.497417:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175701.497420:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.497423:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175701.497425:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175701.497428:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175701.497434:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786175701.497437:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175701.497446:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856162176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175701.497459:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175701.497464:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175701.497470:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856162176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175701.497493:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175701.497497:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175701.497501:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175701.497504:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175701.497507:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175701.497536:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175701.497542:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 5924
00000400:00000200:2.0:1786175701.497548:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175701.497554:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175701.497558:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175701.497562:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497567:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175701.497570:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497574:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175701.497579:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175701.497582:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175701.497584:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175701.497587:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175701.497591:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175701.497595:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175701.497602:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497604:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175701.497609:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497611:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175701.497615:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175701.497620:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175701.497625:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175701.497628:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175701.497632:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175701.497635:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.497638:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175701.497641:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175701.497646:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.497648:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175701.497655:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786175701.497659:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175701.497664:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175701.497667:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175701.497669:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175701.497672:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175701.497676:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175701.497679:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786175701.497683:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175701.497690:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856162304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175701.497705:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175701.497707:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175701.497710:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856162304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175701.497725:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175701.497732:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (5934)
00000800:00000200:1.1:1786175701.497827:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175701.497829:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786175701.498058:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175701.498059:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786175701.498349:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175701.498351:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786175701.498647:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175701.498650:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1786175706.617708:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175706.617722:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786175706.619123:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175706.619127:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786175706.619557:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175706.619561:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786175706.619909:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175706.619912:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786175706.620412:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175706.620416:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786175711.737580:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175711.737595:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786175711.738801:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175711.738805:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786175711.739194:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175711.739197:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786175711.739606:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175711.739612:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786175711.740013:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175711.740017:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786175716.857780:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175716.857792:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786175716.858991:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175716.858994:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786175716.859408:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175716.859412:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786175716.859855:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175716.859859:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786175716.860305:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175716.860309:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786175721.977691:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786175721.977703:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786175721.979195:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175721.979202:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786175721.979640:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175721.979645:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786175721.980094:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175721.980097:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786175721.980723:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175721.980727:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786175727.097796:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175727.097801:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786175727.099061:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175727.099065:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786175727.099452:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175727.099455:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786175727.099814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175727.099817:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786175727.100350:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175727.100353:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786175732.217548:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175732.217558:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786175732.218725:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175732.218728:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786175732.219110:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175732.219113:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786175732.219462:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175732.219466:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786175732.219947:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175732.219950:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1786175737.336762:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175737.336770:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175737.336773:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175737.336780:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175737.336789:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175737.336797:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175737.336803:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175737.336809:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175737.336815:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175737.337844:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175737.338586:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175737.339224:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175737.339609:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175737.339920:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786175738.360634:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175738.360643:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175738.360646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175738.360652:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175738.360657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175738.360661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175738.360664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175738.360668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175738.360672:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175739.384651:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175739.384664:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175739.384667:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175739.384676:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175739.384682:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175739.384688:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175739.384693:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175739.384698:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175739.384704:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175739.960912:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175739.960917:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175739.960921:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175740.408591:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175740.408598:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175740.408600:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175740.408605:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175740.408608:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175740.408611:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175740.408614:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175740.408616:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175740.408618:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175741.432627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175741.432638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175741.432641:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175741.432649:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175741.432654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175741.432660:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175741.432664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175741.432668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175741.432671:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175742.456635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175742.456644:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175742.456659:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175742.456668:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175742.456675:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175742.456683:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175742.456692:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175742.456699:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175742.456709:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175742.457867:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175742.457877:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786175742.459171:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175742.459175:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786175742.459603:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175742.459606:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786175742.459946:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175742.459950:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786175742.460320:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175742.460322:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:2.0:1786175743.032665:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175743.032668:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175743.032671:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175743.480580:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175743.480586:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175743.480588:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175743.480593:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175743.480596:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175743.480598:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175743.480601:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175743.480603:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175743.480605:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175746.104713:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175746.104716:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175746.104719:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175747.576723:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175747.576730:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175747.576733:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175747.576738:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175747.576742:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175747.576747:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175747.576750:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175747.576754:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175747.576758:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175747.577356:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175747.577362:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1786175747.577899:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175747.577901:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1786175747.578185:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175747.578188:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1786175747.578461:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175747.578463:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1786175747.578672:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175747.578673:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000100:00000001:2.0:1786175748.600606:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175748.600614:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175748.600617:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175748.600623:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175748.600627:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175748.600631:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175748.600634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175748.600637:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175748.600641:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175749.176809:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175749.176816:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175749.176821:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175749.624602:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175749.624611:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175749.624613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175749.624620:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175749.624625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175749.624657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175749.624662:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175749.624666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175749.624671:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175750.648601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175750.648610:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175750.648613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175750.648620:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175750.648624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175750.648628:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175750.648632:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175750.648635:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175750.648638:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175751.672620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175751.672627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175751.672629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175751.672634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175751.672637:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175751.672640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175751.672642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175751.672644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175751.672646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175752.248898:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175752.248908:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175752.248916:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175752.696590:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175752.696600:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175752.696603:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175752.696608:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175752.696616:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175752.696620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175752.696623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175752.696625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175752.696627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175752.697543:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175752.697548:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1786175752.698421:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175752.698425:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1786175752.698833:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175752.698836:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1786175752.699113:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175752.699115:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1786175752.699550:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175752.699552:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000020:00000001:2.0:1786175755.320793:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175755.320797:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175755.320800:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786175757.817886:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175757.817903:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786175757.818866:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175757.818869:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786175757.819247:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175757.819250:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786175757.819600:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175757.819604:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786175757.819976:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175757.819980:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:2.0:1786175758.392736:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175758.392740:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175758.392743:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175761.464755:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175761.464758:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175761.464761:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786175762.937884:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175762.937892:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1786175762.939215:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175762.939218:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1786175762.939652:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175762.939655:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1786175762.940154:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175762.940160:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1786175762.941088:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175762.941094:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:2.0:1786175764.536856:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175764.536868:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175764.536876:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175767.608725:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175767.608730:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175767.608733:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786175768.057438:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175768.057443:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786175768.058459:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175768.058462:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786175768.058808:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175768.058810:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786175768.059241:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175768.059244:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786175768.059502:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175768.059505:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:2.0:1786175770.680764:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175770.680773:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175770.680780:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175773.176707:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175773.176715:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175773.176722:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175773.176731:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175773.176741:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175773.176751:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175773.176762:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175773.176771:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175773.176777:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175773.177685:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175773.177698:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786175773.178754:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175773.178757:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786175773.179126:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175773.179129:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786175773.179481:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175773.179485:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786175773.180116:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175773.180119:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:2.0:1786175773.752823:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175773.752828:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175773.752832:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175774.200682:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175774.200690:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175774.200693:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175774.200699:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175774.200703:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175774.200706:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175774.200709:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175774.200713:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175774.200716:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175775.224600:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175775.224607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175775.224609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175775.224614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175775.224616:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175775.224620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175775.224622:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175775.224624:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175775.224627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175776.248642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175776.248651:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175776.248654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175776.248661:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175776.248665:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175776.248670:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175776.248673:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175776.248676:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175776.248680:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175776.824816:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175776.824821:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175776.824825:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175777.272609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175777.272616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175777.272619:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175777.272624:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175777.272628:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175777.272631:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175777.272634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175777.272637:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175777.272640:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175778.296664:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175778.296685:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175778.296698:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175778.296710:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175778.296724:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175778.296739:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6001
00000400:00000200:2.0:1786175778.296756:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175778.296767:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175778.296771:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175778.296776:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.296781:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175778.296785:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.296788:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175778.296793:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175778.296796:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175778.296801:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175778.296805:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175778.296808:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175778.296812:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175778.296815:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.296818:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175778.296821:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.296823:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175778.296838:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175778.296842:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175778.296845:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175778.296850:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175778.296854:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175778.296859:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.296861:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175778.296864:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175778.296879:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786175778.296885:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175778.296891:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175778.296893:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.296898:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175778.296902:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175778.296922:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175778.296934:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786175778.296944:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175778.296957:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856171392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175778.296973:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175778.296980:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175778.296986:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856171392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175778.297016:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175778.297021:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175778.297026:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175778.297028:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175778.297030:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175778.297035:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175778.297039:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6001
00000400:00000200:2.0:1786175778.297045:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175778.297049:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175778.297052:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175778.297057:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.297061:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175778.297071:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.297080:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175778.297085:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175778.297087:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175778.297091:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175778.297096:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175778.297104:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175778.297117:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175778.297139:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.297164:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175778.297176:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.297187:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175778.297205:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175778.297222:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175778.297230:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175778.297250:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175778.297272:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175778.297294:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.297306:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175778.297313:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175778.297320:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.297327:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175778.297335:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786175778.297341:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175778.297346:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175778.297356:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.297361:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175778.297366:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175778.297373:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175778.297396:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000001:2.0:1786175778.297406:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175778.297431:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856171520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175778.297466:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175778.297471:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175778.297557:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856171520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175778.297575:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175778.297581:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175778.297592:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175778.297600:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175778.297605:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175778.297611:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175778.297621:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6001
00000400:00000200:2.0:1786175778.297638:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175778.297650:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175778.297661:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175778.297671:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.297684:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175778.297693:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.297708:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175778.297714:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175778.297721:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175778.297729:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175778.297737:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175778.297751:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175778.297761:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175778.297780:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.297788:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175778.297802:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.297807:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175778.297818:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175778.297835:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175778.297841:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175778.297847:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1786175778.297850:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.1:1786175778.297857:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
02000000:00000001:2.0:1786175778.297861:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175778.297867:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.297885:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175778.297899:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175778.297904:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.297908:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175778.297914:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786175778.297921:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175778.297933:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175778.297937:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.297942:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175778.297946:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175778.297959:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175778.297973:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786175778.297981:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175778.298004:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856171648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175778.298028:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175778.298040:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175778.298062:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856171648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175778.298092:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175778.298106:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175778.298120:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175778.298124:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175778.298131:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175778.298137:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175778.298145:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6001
00000400:00000200:2.0:1786175778.298153:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175778.298169:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175778.298174:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175778.298188:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.298194:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175778.298205:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.298222:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175778.298235:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175778.298242:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175778.298249:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175778.298258:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175778.298272:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175778.298283:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175778.298303:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.298311:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175778.298319:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.298325:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175778.298339:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175778.298360:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175778.298378:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175778.298386:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175778.298396:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175778.298402:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.298408:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175778.298414:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175778.298420:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.298431:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175778.298444:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786175778.298453:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175778.298457:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175778.298462:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.298503:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175778.298547:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175778.298552:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175778.298561:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786175778.298582:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175778.298603:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856171776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175778.298709:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175778.298720:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175778.298738:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856171776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175778.298839:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175778.298851:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175778.298876:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175778.298888:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175778.298896:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175778.298914:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175778.298934:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6001
00000400:00000200:2.0:1786175778.298952:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175778.298964:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175778.298972:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175778.298987:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.299008:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175778.299023:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.299040:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175778.299057:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175778.299067:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175778.299078:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175778.299091:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175778.299105:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175778.299120:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175778.299142:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.299157:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175778.299174:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175778.299190:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175778.299210:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175778.299230:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175778.299248:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175778.299264:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175778.299282:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175778.299299:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.299313:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175778.299327:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175778.299345:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.299361:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175778.299381:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:2.0:1786175778.299384:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000800:00000200:0.1:1786175778.299385:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000040:2.0:1786175778.299403:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175778.299419:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175778.299427:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175778.299440:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175778.299448:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175778.299458:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175778.299506:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786175778.299579:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175778.299599:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856171904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175778.299655:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175778.299668:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175778.299696:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856171904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175778.299787:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175778.299809:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6011)
00000800:00000200:0.1:1786175778.299994:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175778.299996:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786175778.300246:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175778.300248:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786175778.300752:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175778.300754:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:2.0:1786175779.896708:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175779.896715:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175779.896720:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175782.968812:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175782.968819:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175782.968825:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786175783.416641:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175783.416667:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175783.416669:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175783.416674:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175783.416681:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175783.416692:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6006
00000400:00000200:2.0:1786175783.416704:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175783.416711:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175783.416714:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175783.416720:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.416724:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175783.416728:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.416732:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175783.416738:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175783.416741:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175783.416744:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175783.416747:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175783.416751:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175783.416754:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175783.416758:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.416772:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175783.416780:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.416783:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175783.416787:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175783.416790:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175783.416793:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175783.416795:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175783.416799:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175783.416802:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.416804:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175783.416806:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175783.416815:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000020:00000040:2.0:1786175783.416818:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175783.416822:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175783.416824:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.416827:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175783.416830:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175783.416834:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175783.416841:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786175783.416847:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175783.416856:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856172032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175783.416865:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175783.416869:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175783.416872:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856172032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175783.416893:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175783.416896:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175783.416899:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175783.416901:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175783.416902:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175783.416905:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175783.416907:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6006
00000400:00000200:2.0:1786175783.416911:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175783.416913:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175783.416916:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175783.416918:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.416920:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175783.416922:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.416925:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175783.416927:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175783.416928:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175783.416929:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175783.416931:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175783.416933:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175783.416935:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175783.416939:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.416943:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175783.416948:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.416952:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175783.416954:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175783.416960:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175783.416967:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175783.416971:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175783.416984:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175783.416990:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.416994:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175783.417000:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175783.417007:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.417011:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175783.417015:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000020:00000040:2.0:1786175783.417019:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175783.417023:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175783.417026:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.417028:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175783.417030:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175783.417035:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175783.417042:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786175783.417047:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175783.417055:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856172160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175783.417077:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175783.417080:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175783.417090:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856172160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175783.417105:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175783.417111:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175783.417121:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175783.417125:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175783.417128:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175783.417133:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175783.417140:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6006
00000400:00000200:2.0:1786175783.417146:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175783.417153:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175783.417157:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175783.417160:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417165:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175783.417177:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417183:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175783.417188:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175783.417192:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175783.417196:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175783.417200:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175783.417206:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175783.417212:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175783.417221:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417226:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175783.417230:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417233:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175783.417238:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175783.417245:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417249:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175783.417251:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175783.417273:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175783.417280:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.417284:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175783.417288:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175783.417292:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.417298:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175783.417303:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000020:00000040:2.0:1786175783.417310:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175783.417317:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175783.417321:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.417325:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175783.417328:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175783.417332:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175783.417349:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000800:00000200:0.1:1786175783.417357:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175783.417365:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1786175783.417367:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175783.417380:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856172288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175783.417396:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175783.417403:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175783.417415:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856172288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175783.417450:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175783.417456:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175783.417463:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175783.417467:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175783.417498:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175783.417504:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175783.417511:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6006
00000400:00000200:2.0:1786175783.417518:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175783.417524:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175783.417549:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175783.417554:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417555:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175783.417557:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417559:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175783.417561:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175783.417562:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175783.417564:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175783.417567:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175783.417570:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175783.417573:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175783.417579:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417587:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175783.417595:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417602:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175783.417607:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175783.417616:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417624:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175783.417630:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175783.417636:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175783.417638:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.417639:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175783.417642:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175783.417644:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.417645:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175783.417649:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000020:00000040:2.0:1786175783.417653:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175783.417656:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175783.417658:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.417663:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175783.417664:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175783.417666:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175783.417680:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786175783.417686:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175783.417697:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856172416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175783.417723:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175783.417729:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175783.417744:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856172416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175783.417768:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175783.417774:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175783.417782:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175783.417785:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175783.417787:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175783.417796:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175783.417803:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6006
00000400:00000200:2.0:1786175783.417815:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175783.417828:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175783.417833:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175783.417840:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417844:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175783.417849:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417853:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175783.417859:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175783.417862:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175783.417880:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175783.417897:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175783.417901:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175783.417904:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175783.417912:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417918:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175783.417923:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417929:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175783.417937:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175783.417945:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175783.417968:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175783.417974:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175783.417978:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175783.417984:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.417988:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175783.417992:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175783.417998:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.418003:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175783.418012:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786175783.418017:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175783.418023:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175783.418026:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175783.418031:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175783.418034:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175783.418038:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175783.418047:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786175783.418053:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175783.418062:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856172544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175783.418087:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175783.418092:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175783.418097:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856172544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175783.418114:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175783.418120:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6016)
00000800:00000200:0.1:1786175783.418292:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175783.418297:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786175783.418727:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175783.418729:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786175783.419096:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175783.419099:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786175783.419377:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175783.419380:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:2.0:1786175786.040894:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175786.040903:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175786.040909:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786175788.537235:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175788.537241:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786175788.538005:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175788.538007:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786175788.538238:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175788.538239:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786175788.538482:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175788.538485:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786175788.538725:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175788.538727:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:2.0:1786175789.112690:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175789.112694:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175789.112698:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175792.184785:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175792.184791:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175792.184796:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786175793.656767:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175793.656776:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175793.656779:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175793.656784:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175793.656790:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175793.656799:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6016
00000400:00000200:2.0:1786175793.656810:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175793.656819:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175793.656824:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175793.656829:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.656833:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175793.656838:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.656842:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175793.656847:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175793.656850:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175793.656854:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175793.656858:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175793.656862:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175793.656865:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175793.656871:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.656874:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175793.656877:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.656880:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175793.656884:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175793.656890:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175793.656894:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175793.656899:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175793.656904:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175793.656907:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.656910:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175793.656913:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175793.656924:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786175793.656929:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175793.656934:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175793.656936:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.656941:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175793.656944:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175793.656950:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175793.656959:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:2.0:1786175793.656967:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175793.656980:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856173312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175793.656994:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175793.657000:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175793.657006:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856173312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175793.657031:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175793.657035:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175793.657040:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175793.657042:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175793.657044:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175793.657048:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175793.657052:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6016
00000400:00000200:2.0:1786175793.657057:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175793.657061:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175793.657065:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175793.657069:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.657088:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175793.657094:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.657100:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175793.657105:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175793.657109:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175793.657115:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175793.657120:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175793.657127:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175793.657133:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175793.657144:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.657149:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175793.657157:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.657164:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175793.657170:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175793.657181:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175793.657188:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175793.657194:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175793.657205:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175793.657211:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.657239:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175793.657248:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175793.657254:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.657261:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175793.657270:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786175793.657276:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175793.657279:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175793.657282:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.657305:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175793.657307:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175793.657312:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175793.657368:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786175793.657380:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175793.657397:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856173440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175793.657420:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175793.657428:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175793.657438:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856173440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175793.657454:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175793.657458:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175793.657469:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175793.657475:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175793.657480:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175793.657484:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175793.657492:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6016
00000400:00000200:2.0:1786175793.657505:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175793.657558:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175793.657562:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175793.657567:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.657570:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175793.657573:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.657576:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175793.657579:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175793.657582:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175793.657588:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175793.657593:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175793.657600:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175793.657608:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175793.657619:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.657625:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175793.657635:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.657639:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175793.657650:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175793.657660:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175793.657666:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175793.657671:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175793.657679:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175793.657689:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.657694:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175793.657701:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175793.657710:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.657713:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175793.657721:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786175793.657732:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175793.657743:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175793.657747:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.657756:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175793.657760:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175793.657766:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175793.657767:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175793.657775:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
02000000:00000010:2.0:1786175793.657776:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786175793.657785:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175793.657801:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856173568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175793.657822:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175793.657830:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175793.657842:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856173568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175793.657861:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175793.657867:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175793.657871:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175793.657878:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175793.657881:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175793.657888:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175793.657895:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6016
00000400:00000200:2.0:1786175793.657908:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175793.657916:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175793.657925:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175793.657936:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.657941:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175793.657950:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.657956:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175793.657964:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175793.657968:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175793.657973:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175793.657978:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175793.657985:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175793.657990:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175793.658002:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.658008:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175793.658013:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.658023:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175793.658031:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175793.658042:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175793.658051:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175793.658053:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175793.658058:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175793.658063:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.658066:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175793.658069:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175793.658076:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.658079:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175793.658086:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786175793.658092:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175793.658100:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175793.658107:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.658111:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175793.658117:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175793.658122:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175793.658133:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786175793.658142:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175793.658158:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856173696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175793.658176:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175793.658183:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175793.658194:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856173696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175793.658217:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175793.658224:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175793.658238:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175793.658243:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175793.658249:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175793.658255:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175793.658265:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6016
00000400:00000200:2.0:1786175793.658280:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175793.658289:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175793.658297:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175793.658309:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.658318:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175793.658352:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.658361:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175793.658370:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175793.658376:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175793.658383:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175793.658388:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175793.658395:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175793.658405:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175793.658418:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.658426:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175793.658434:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175793.658443:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175793.658451:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175793.658460:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175793.658467:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175793.658472:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175793.658479:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175793.658485:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.658492:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175793.658497:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175793.658504:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.658538:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175793.658544:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786175793.658548:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175793.658555:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175793.658558:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175793.658567:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175793.658570:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175793.658577:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175793.658586:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786175793.658596:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175793.658611:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856173824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175793.658632:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175793.658639:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175793.658654:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856173824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175793.658678:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175793.658691:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6027)
00000800:00000200:0.1:1786175793.658966:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175793.658970:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786175793.659370:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175793.659373:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786175793.659813:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175793.659818:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786175793.660222:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175793.660225:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:2.0:1786175795.256770:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175795.256776:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175795.256781:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175798.328834:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175798.328839:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175798.328843:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786175798.776637:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175798.776655:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175798.776665:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175798.776674:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175798.776686:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175798.776701:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6022
00000400:00000200:2.0:1786175798.776720:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175798.776730:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175798.776734:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175798.776739:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.776743:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175798.776747:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.776749:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175798.776754:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175798.776757:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175798.776760:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175798.776764:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175798.776767:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175798.776770:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175798.776773:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.776776:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175798.776790:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.776793:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175798.776797:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175798.776819:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175798.776823:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175798.776827:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175798.776831:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175798.776834:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.776837:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175798.776840:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175798.776851:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786175798.776855:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175798.776860:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175798.776862:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.776866:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175798.776870:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175798.776875:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175798.776884:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786175798.776892:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175798.776903:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856173952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175798.776916:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175798.776922:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175798.776927:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856173952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175798.776955:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175798.776976:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175798.776980:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175798.776982:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175798.776984:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175798.776988:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175798.776990:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6022
00000400:00000200:2.0:1786175798.776995:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175798.777000:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175798.777003:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175798.777007:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777011:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175798.777014:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777017:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175798.777020:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175798.777021:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175798.777023:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175798.777026:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175798.777034:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175798.777042:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175798.777049:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777055:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175798.777060:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777068:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175798.777077:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175798.777086:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777095:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175798.777103:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175798.777111:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175798.777114:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.777118:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175798.777123:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175798.777130:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.777135:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175798.777145:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000020:00000040:2.0:1786175798.777153:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175798.777170:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175798.777175:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.777179:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175798.777182:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175798.777185:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175798.777195:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786175798.777205:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175798.777219:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856174080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175798.777234:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175798.777237:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175798.777247:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856174080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175798.777266:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175798.777271:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175798.777304:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175798.777311:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175798.777318:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175798.777329:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175798.777337:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6022
00000400:00000200:2.0:1786175798.777347:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175798.777359:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175798.777363:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175798.777367:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777376:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175798.777380:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777386:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175798.777391:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175798.777393:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175798.777395:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175798.777397:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175798.777403:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175798.777407:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175798.777411:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777413:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175798.777416:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777420:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175798.777428:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175798.777431:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777434:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175798.777442:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175798.777449:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175798.777453:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.777458:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175798.777465:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175798.777471:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.777475:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175798.777483:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000020:00000040:2.0:1786175798.777491:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175798.777499:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175798.777505:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.777564:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175798.777572:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175798.777577:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175798.777587:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786175798.777599:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175798.777614:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856174208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175798.777655:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175798.777663:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175798.777684:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856174208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1786175798.777685:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175798.777694:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:2.0:1786175798.777710:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175798.777720:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175798.777734:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175798.777741:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175798.777751:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175798.777761:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175798.777774:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6022
00000400:00000200:2.0:1786175798.777793:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175798.777807:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175798.777811:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175798.777820:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777829:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175798.777836:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777843:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175798.777851:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175798.777857:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175798.777864:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175798.777870:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175798.777877:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175798.777885:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175798.777901:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777905:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175798.777909:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777919:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175798.777928:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175798.777939:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175798.777945:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175798.777948:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175798.777976:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175798.777981:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.777987:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175798.777992:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175798.778001:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.778005:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175798.778016:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786175798.778022:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175798.778030:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175798.778035:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.778041:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175798.778044:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175798.778048:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175798.778056:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786175798.778067:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175798.778079:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856174336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175798.778101:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175798.778108:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175798.778121:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856174336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175798.778139:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175798.778147:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175798.778152:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175798.778157:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175798.778162:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175798.778168:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175798.778175:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6022
00000400:00000200:2.0:1786175798.778189:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175798.778198:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175798.778204:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175798.778210:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.778215:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175798.778221:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.778224:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175798.778231:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175798.778235:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175798.778239:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175798.778243:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175798.778249:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175798.778255:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175798.778264:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.778268:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175798.778273:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175798.778302:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175798.778311:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175798.778327:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175798.778335:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175798.778343:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175798.778352:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175798.778359:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.778364:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175798.778372:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175798.778380:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.778384:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175798.778391:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786175798.778396:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175798.778404:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175798.778407:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175798.778414:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175798.778418:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175798.778423:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175798.778432:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786175798.778440:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175798.778455:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856174464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175798.778499:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175798.778506:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175798.778547:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856174464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175798.778564:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175798.778572:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6032)
00000800:00000200:0.1:1786175798.778820:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175798.778825:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786175798.779219:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175798.779223:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786175798.779704:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175798.779707:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786175798.780135:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175798.780138:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000020:00000001:2.0:1786175801.400793:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175801.400799:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175801.400804:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786175803.896646:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175803.896663:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175803.896673:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175803.896680:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175803.896688:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175803.896702:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6027
00000400:00000200:2.0:1786175803.896719:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175803.896734:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175803.896744:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175803.896751:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.896756:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175803.896760:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.896763:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175803.896769:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175803.896771:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175803.896776:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175803.896781:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175803.896785:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175803.896788:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175803.896793:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.896795:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175803.896798:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.896800:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175803.896804:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175803.896809:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175803.896813:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175803.896817:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175803.896822:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175803.896826:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.896829:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175803.896832:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175803.896846:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786175803.896851:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175803.896858:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175803.896860:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.896864:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175803.896869:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175803.896876:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175803.896888:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786175803.896897:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175803.896911:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856174592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175803.896926:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175803.896932:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175803.896939:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856174592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175803.896976:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175803.896980:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175803.896985:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175803.896987:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175803.896990:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175803.896995:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175803.896998:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6027
00000400:00000200:2.0:1786175803.897004:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175803.897008:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175803.897012:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175803.897018:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.897022:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175803.897027:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.897032:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175803.897040:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175803.897043:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175803.897048:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175803.897052:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175803.897055:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175803.897061:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175803.897082:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.897092:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175803.897102:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.897111:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175803.897126:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175803.897141:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175803.897145:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175803.897153:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175803.897167:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175803.897180:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.897186:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175803.897194:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175803.897216:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.897283:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175803.897294:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786175803.897306:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175803.897319:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175803.897324:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.897332:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175803.897342:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175803.897355:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175803.897371:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000001:2.0:1786175803.897387:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175803.897413:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856174720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175803.897460:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175803.897468:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175803.897476:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856174720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175803.897506:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175803.897546:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175803.897552:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175803.897555:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175803.897558:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175803.897570:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175803.897579:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6027
00000400:00000200:2.0:1786175803.897596:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175803.897612:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175803.897630:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175803.897639:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.897653:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175803.897664:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:0.1:1786175803.897666:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786175803.897671:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175803.897677:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175803.897679:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175803.897682:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175803.897685:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175803.897687:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175803.897698:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175803.897714:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.897720:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175803.897727:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.897739:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175803.897744:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175803.897752:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175803.897757:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175803.897760:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175803.897768:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175803.897776:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.897781:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175803.897788:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175803.897800:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.897808:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175803.897817:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786175803.897829:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175803.897836:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175803.897844:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.897860:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175803.897867:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175803.897873:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175803.897891:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786175803.897905:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175803.897921:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856174848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175803.897960:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175803.897969:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175803.897983:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856174848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175803.898035:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175803.898049:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175803.898068:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175803.898074:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175803.898081:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175803.898095:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175803.898108:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6027
00000400:00000200:2.0:1786175803.898123:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175803.898138:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175803.898148:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175803.898159:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.898170:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175803.898180:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.898185:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175803.898195:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175803.898197:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175803.898203:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175803.898208:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175803.898215:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175803.898220:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175803.898263:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.898276:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175803.898289:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.898303:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175803.898314:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175803.898334:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175803.898362:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175803.898373:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.1:1786175803.898374:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1786175803.898385:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175803.898396:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.898412:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175803.898421:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175803.898434:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.898443:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175803.898465:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786175803.898475:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175803.898488:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175803.898494:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.898504:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175803.898637:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175803.898644:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175803.898658:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786175803.898669:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175803.898684:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856174976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175803.898699:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175803.898706:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175803.898728:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856174976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175803.898750:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175803.898756:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175803.898765:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175803.898768:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175803.898771:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175803.898775:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175803.898782:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6027
00000400:00000200:2.0:1786175803.898799:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175803.898808:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175803.898813:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175803.898819:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.898825:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175803.898830:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.898834:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175803.898840:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175803.898844:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175803.898849:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:0.1:1786175803.898853:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1786175803.898855:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175803.898861:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175803.898866:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175803.898873:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.898875:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175803.898877:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175803.898883:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175803.898890:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175803.898897:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175803.898905:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175803.898910:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175803.898918:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175803.898924:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.898929:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175803.898936:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175803.898942:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.898947:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175803.898954:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786175803.898961:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175803.898969:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175803.898972:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175803.898977:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175803.898980:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175803.898983:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175803.898991:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786175803.898998:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175803.899007:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856175104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175803.899020:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175803.899025:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175803.899036:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856175104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175803.899056:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175803.899065:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6037)
00000800:00000200:0.1:1786175803.899175:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175803.899343:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786175804.472672:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175804.472677:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175804.472681:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175807.544776:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175807.544784:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175807.544791:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786175809.016619:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175809.016637:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175809.016648:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175809.016656:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175809.016662:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175809.016671:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6032
00000400:00000200:2.0:1786175809.016682:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175809.016690:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175809.016693:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175809.016696:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.016700:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175809.016702:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.016705:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175809.016709:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175809.016711:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175809.016715:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175809.016718:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175809.016720:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175809.016722:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175809.016725:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.016727:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175809.016729:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.016731:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175809.016734:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175809.016737:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175809.016740:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175809.016743:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175809.016747:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175809.016750:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.016752:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175809.016754:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175809.016764:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000020:00000040:2.0:1786175809.016768:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175809.016773:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175809.016786:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.016790:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175809.016793:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175809.016799:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175809.016809:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786175809.016817:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175809.016828:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856175232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175809.016840:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175809.016846:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175809.016851:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856175232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175809.016882:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175809.016886:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175809.016889:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175809.016891:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175809.016892:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175809.016895:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175809.016899:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6032
00000400:00000200:2.0:1786175809.016902:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175809.016905:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175809.016907:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175809.016910:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.016912:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175809.016914:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.016916:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175809.016928:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175809.016930:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175809.016932:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175809.016934:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175809.016936:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175809.016939:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175809.016943:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.016944:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175809.016946:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.016947:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175809.016951:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175809.016958:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175809.016962:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175809.016968:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175809.016979:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175809.016986:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.016991:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175809.017010:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175809.017019:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.017029:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175809.017045:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000020:00000040:2.0:1786175809.017057:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175809.017063:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175809.017069:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.017075:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175809.017079:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175809.017086:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175809.017109:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786175809.017119:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175809.017138:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856175360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175809.017160:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175809.017168:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175809.017201:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856175360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175809.017222:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175809.017230:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175809.017243:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175809.017249:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175809.017254:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175809.017264:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175809.017275:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6032
00000400:00000200:2.0:1786175809.017290:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175809.017301:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175809.017307:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175809.017320:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.017326:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175809.017335:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.017342:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175809.017352:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175809.017357:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175809.017362:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175809.017368:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175809.017374:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175809.017389:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175809.017402:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.017413:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175809.017422:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.017429:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175809.017439:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175809.017450:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175809.017459:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175809.017463:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175809.017478:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175809.017489:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.017503:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175809.017509:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175809.017593:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.017601:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175809.017611:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:2.0:1786175809.017614:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000800:00000200:0.1:1786175809.017620:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000040:2.0:1786175809.017623:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175809.017630:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175809.017635:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.017643:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175809.017648:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175809.017655:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175809.017667:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786175809.017678:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175809.017694:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856175488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175809.017726:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175809.017735:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175809.017749:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856175488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175809.017763:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175809.017773:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175809.017784:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175809.017788:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175809.017791:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175809.017806:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175809.017817:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6032
00000400:00000200:2.0:1786175809.017837:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175809.017853:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175809.017865:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175809.017878:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.017890:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175809.017900:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.017906:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175809.017913:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175809.017918:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175809.017925:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175809.017932:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175809.017938:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175809.017945:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175809.017954:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.017961:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175809.017966:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.017975:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175809.017985:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175809.017996:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175809.018004:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175809.018011:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175809.018021:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175809.018029:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.018038:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175809.018047:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175809.018055:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.018062:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175809.018070:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000020:00000040:2.0:1786175809.018078:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175809.018086:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175809.018092:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.018098:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175809.018105:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175809.018110:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175809.018140:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786175809.018154:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175809.018166:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856175616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175809.018208:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175809.018216:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175809.018238:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856175616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175809.018263:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175809.018277:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175809.018286:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175809.018293:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175809.018297:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175809.018306:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175809.018315:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6032
00000400:00000200:2.0:1786175809.018330:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175809.018335:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175809.018340:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175809.018345:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.018350:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175809.018356:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.018364:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175809.018369:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175809.018373:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175809.018378:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175809.018381:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175809.018387:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175809.018390:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175809.018399:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.018405:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175809.018410:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175809.018418:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175809.018424:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175809.018433:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175809.018438:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175809.018441:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175809.018446:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175809.018452:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.018459:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175809.018465:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175809.018476:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.018491:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175809.018504:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786175809.018546:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175809.018556:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175809.018563:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175809.018568:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175809.018577:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175809.018589:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175809.018607:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786175809.018622:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175809.018630:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856175744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175809.018665:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175809.018669:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175809.018677:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856175744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175809.018743:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175809.018755:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6042)
00000800:00000200:0.1:1786175809.018903:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175809.018905:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786175809.019123:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175809.019125:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786175809.019330:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175809.019332:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786175809.019597:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175809.019599:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1786175810.040705:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175810.040715:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175810.040718:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175810.040724:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175810.040729:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175810.040734:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175810.040737:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175810.040740:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175810.040743:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175810.616750:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175810.616776:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175810.616783:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175811.064628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175811.064656:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175811.064661:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175811.064669:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175811.064687:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175811.064692:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175811.064697:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175811.064702:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175811.064707:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175812.088644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175812.088654:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175812.088657:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175812.088665:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175812.088668:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175812.088672:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175812.088675:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175812.088678:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175812.088680:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175813.112615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175813.112626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175813.112629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175813.112649:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175813.112655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175813.112659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175813.112663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175813.112667:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175813.112671:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175813.688798:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175813.688803:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175813.688807:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175814.136655:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175814.136665:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175814.136667:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175814.136673:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175814.136678:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175814.136682:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175814.136686:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175814.136690:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175814.136693:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175814.137346:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175814.137355:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175814.137895:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175814.137897:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175814.138156:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175814.138158:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175814.138379:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175814.138381:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786175814.138596:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175814.138597:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1786175815.160612:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175815.160619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175815.160620:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175815.160625:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175815.160628:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175815.160631:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175815.160633:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175815.160635:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175815.160637:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175816.184599:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175816.184605:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175816.184607:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175816.184612:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175816.184615:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175816.184618:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175816.184620:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175816.184622:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175816.184624:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175816.760779:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175816.760785:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175816.760789:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175817.208623:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175817.208633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175817.208636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175817.208643:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175817.208648:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175817.208653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175817.208656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175817.208659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175817.208663:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175818.232627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175818.232634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175818.232636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175818.232641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175818.232644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175818.232648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175818.232650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175818.232652:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175818.232655:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175819.256596:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175819.256621:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175819.256624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175819.256631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175819.256635:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175819.256639:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175819.256642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175819.256647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175819.256651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175819.257383:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175819.257389:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786175819.258162:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175819.258164:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786175819.258390:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175819.258392:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786175819.258683:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175819.258684:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786175819.258880:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175819.258882:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:2.0:1786175819.832779:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175819.832784:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175819.832789:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175820.280712:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175820.280722:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175820.280726:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175820.280733:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175820.280738:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175820.280743:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175820.280746:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175820.280749:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175820.280753:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175821.304576:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175821.304582:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175821.304584:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175821.304588:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175821.304591:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175821.304594:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175821.304596:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175821.304598:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175821.304600:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175822.328623:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175822.328632:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175822.328634:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175822.328641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175822.328647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175822.328652:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175822.328656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175822.328660:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175822.328664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175822.904719:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175822.904725:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175822.904729:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175823.352659:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175823.352671:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175823.352674:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175823.352682:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175823.352687:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175823.352692:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175823.352696:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175823.352700:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175823.352705:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175824.376684:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175824.376693:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175824.376695:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175824.376702:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175824.376710:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175824.376716:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175824.376720:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175824.376725:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175824.376731:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175824.377853:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175824.377871:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786175824.378905:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175824.378907:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786175824.379350:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175824.379353:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786175824.379828:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175824.379832:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786175824.380194:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175824.380196:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1786175825.400665:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175825.400677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175825.400681:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175825.400690:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175825.400697:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175825.400703:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175825.400708:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175825.400713:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175825.400718:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175825.976713:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175825.976718:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175825.976721:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175826.424625:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175826.424633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175826.424636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175826.424641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175826.424644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175826.424647:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175826.424649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175826.424651:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175826.424654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175827.448615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175827.448622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175827.448623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175827.448628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175827.448631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175827.448633:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175827.448635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175827.448637:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175827.448639:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175828.472582:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175828.472588:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175828.472589:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175828.472594:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175828.472596:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175828.472599:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175828.472601:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175828.472603:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175828.472605:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175829.048870:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175829.048874:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175829.048878:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175829.496633:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175829.496642:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175829.496645:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175829.496651:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175829.496657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175829.496662:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175829.496666:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175829.496669:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175829.496673:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175829.497438:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175829.497444:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786175829.498441:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175829.498445:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786175829.498928:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175829.498931:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786175829.499303:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175829.499306:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786175829.499688:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175829.499691:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:2.0:1786175830.520608:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175830.520618:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175830.520620:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175830.520626:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175830.520630:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175830.520634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175830.520638:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175830.520642:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175830.520647:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175831.544601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175831.544612:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175831.544614:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175831.544622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175831.544627:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175831.544632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175831.544636:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175831.544639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175831.544644:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175832.120691:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175832.120694:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175832.120697:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175832.568604:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175832.568614:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175832.568617:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175832.568626:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175832.568629:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175832.568633:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175832.568636:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175832.568639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175832.568642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175833.592648:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175833.592660:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175833.592663:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175833.592673:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175833.592678:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175833.592693:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175833.592696:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175833.592700:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175833.592704:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175834.616592:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175834.616599:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175834.616602:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175834.616608:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175834.616614:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175834.616618:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175834.616623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175834.616626:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175834.616631:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175834.617286:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175834.617291:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786175834.617894:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175834.617896:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786175834.618174:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175834.618177:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786175834.618388:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175834.618389:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786175834.618708:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175834.618711:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:2.0:1786175835.192645:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175835.192649:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175835.192652:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175835.640659:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175835.640675:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175835.640680:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175835.640691:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175835.640698:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175835.640704:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175835.640708:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175835.640712:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175835.640716:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175836.664652:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175836.664665:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175836.664669:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175836.664679:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175836.664686:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175836.664692:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175836.664697:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175836.664703:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175836.664721:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175837.688657:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175837.688667:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175837.688669:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175837.688677:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175837.688682:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175837.688686:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175837.688689:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175837.688692:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175837.688696:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175838.264790:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175838.264796:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175838.264801:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175838.712677:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175838.712685:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175838.712687:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175838.712694:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175838.712698:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175838.712702:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175838.712704:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175838.712707:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175838.712710:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175839.736620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175839.736627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175839.736629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175839.736635:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175839.736639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175839.736643:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175839.736646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175839.736649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175839.736652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175839.737413:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175839.737420:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175839.738475:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175839.738482:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175839.738977:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175839.738979:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175839.739334:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175839.739337:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175839.739654:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175839.739657:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:2.0:1786175840.760588:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175840.760595:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175840.760596:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175840.760601:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175840.760604:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175840.760607:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175840.760609:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175840.760611:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175840.760613:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175841.336671:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175841.336675:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175841.336678:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175841.784603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175841.784610:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175841.784612:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175841.784618:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175841.784622:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175841.784625:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175841.784627:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175841.784630:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175841.784648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175842.808636:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175842.808645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175842.808648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175842.808655:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175842.808660:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175842.808665:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175842.808668:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175842.808672:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175842.808676:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175843.832656:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175843.832669:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175843.832672:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175843.832679:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175843.832685:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175843.832690:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175843.832694:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175843.832698:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175843.832702:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175844.408677:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175844.408680:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175844.408683:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175844.856590:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175844.856596:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175844.856598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175844.856604:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175844.856609:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175844.856614:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175844.856620:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175844.856622:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175844.856626:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175844.857182:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175844.857186:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786175844.857858:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175844.857861:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786175844.858193:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175844.858196:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786175844.858477:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175844.858480:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786175844.858919:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175844.858921:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1786175845.880622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175845.880630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175845.880633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175845.880638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175845.880643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175845.880646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175845.880649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175845.880652:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175845.880655:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175846.904630:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175846.904637:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175846.904639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175846.904644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175846.904647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175846.904650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175846.904652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175846.904655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175846.904657:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175847.480651:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175847.480654:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175847.480657:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175847.928619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175847.928625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175847.928627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175847.928631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175847.928634:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175847.928636:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175847.928638:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175847.928640:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175847.928642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175848.952616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175848.952624:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175848.952627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175848.952634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175848.952638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175848.952642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175848.952645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175848.952648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175848.952651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175849.976584:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175849.976590:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175849.976593:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175849.976601:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175849.976604:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175849.976607:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175849.976609:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175849.976611:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175849.976614:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175849.977277:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175849.977285:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175849.977870:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175849.977873:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175849.978123:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175849.978125:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175849.978385:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175849.978388:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175849.978615:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175849.978616:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:2.0:1786175850.552666:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175850.552671:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175850.552675:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175851.000658:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175851.000664:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175851.000666:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175851.000670:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175851.000673:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175851.000676:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175851.000678:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175851.000680:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175851.000682:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175852.024600:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175852.024607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175852.024609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175852.024615:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175852.024618:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175852.024621:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175852.024623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175852.024626:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175852.024628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175853.048609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175853.048616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175853.048617:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175853.048622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175853.048624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175853.048627:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175853.048629:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175853.048639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175853.048641:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175853.624669:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175853.624673:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175853.624676:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175854.072601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175854.072608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175854.072610:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175854.072614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175854.072617:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175854.072620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175854.072622:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175854.072624:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175854.072625:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175855.096633:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175855.096640:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175855.096643:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175855.096650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175855.096657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175855.096664:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175855.096670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175855.096676:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175855.096683:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175855.097311:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175855.097315:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175855.098034:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175855.098036:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175855.098225:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175855.098226:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175855.098448:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175855.098466:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175855.098860:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175855.098862:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1786175856.120583:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175856.120591:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175856.120593:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175856.120597:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175856.120601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175856.120604:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175856.120606:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175856.120608:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175856.120610:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175856.696706:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175856.696710:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175856.696714:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175857.144672:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175857.144679:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175857.144681:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175857.144687:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175857.144690:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175857.144694:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175857.144697:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175857.144699:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175857.144702:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175858.168617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175858.168626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175858.168628:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175858.168634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175858.168638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175858.168642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175858.168645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175858.168648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175858.168662:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175859.192638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175859.192646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175859.192648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175859.192653:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175859.192658:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175859.192662:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175859.192665:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175859.192668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175859.192671:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175859.768714:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175859.768717:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175859.768721:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175860.216561:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175860.216568:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175860.216570:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175860.216593:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175860.216597:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175860.216600:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175860.216602:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175860.216604:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175860.216606:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175860.217301:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175860.217309:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786175860.218068:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175860.218070:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786175860.218332:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175860.218334:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786175860.218576:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175860.218578:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786175860.218932:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175860.218934:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:2.0:1786175861.240605:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175861.240611:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175861.240612:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175861.240617:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175861.240620:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175861.240623:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175861.240625:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175861.240627:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175861.240629:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175862.264607:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175862.264615:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175862.264618:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175862.264624:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175862.264629:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175862.264632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175862.264635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175862.264639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175862.264642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175862.840727:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175862.840732:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175862.840736:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175863.288622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175863.288630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175863.288632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175863.288638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175863.288641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175863.288644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175863.288646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175863.288648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175863.288650:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175864.312622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175864.312630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175864.312632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175864.312636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175864.312640:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175864.312704:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175864.312707:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175864.312710:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175864.312713:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175865.336736:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175865.336746:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175865.336752:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175865.336763:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175865.336773:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175865.336781:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175865.336792:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175865.336799:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175865.336808:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175865.337888:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175865.337902:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175865.339278:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175865.339283:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175865.339963:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175865.339968:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175865.340586:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175865.340592:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175865.341132:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175865.341136:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:2.0:1786175865.912761:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175865.912766:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175865.912770:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175866.360687:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175866.360693:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175866.360694:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175866.360699:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175866.360702:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175866.360705:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175866.360707:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175866.360709:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175866.360711:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175867.384596:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175867.384603:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175867.384605:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175867.384609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175867.384625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175867.384629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175867.384631:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175867.384633:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175867.384635:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175868.408627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175868.408636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175868.408638:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175868.408643:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175868.408648:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175868.408652:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175868.408655:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175868.408659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175868.408662:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175868.984717:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175868.984721:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175868.984723:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175869.432652:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175869.432660:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175869.432662:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175869.432668:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175869.432672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175869.432676:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175869.432680:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175869.432683:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175869.432686:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175870.456602:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175870.456609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175870.456611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175870.456616:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175870.456619:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175870.456622:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175870.456626:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175870.456629:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175870.456631:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175870.457109:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175870.457655:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175870.458021:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175870.458173:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175870.458344:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786175871.480678:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175871.480687:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175871.480689:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175871.480695:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175871.480699:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175871.480704:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175871.480708:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175871.480711:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175871.480714:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175872.056807:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175872.056813:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175872.056818:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175872.504725:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175872.504736:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175872.504739:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175872.504747:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175872.504752:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175872.504757:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175872.504761:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175872.504765:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175872.504769:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175873.528683:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175873.528693:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175873.528696:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175873.528702:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175873.528707:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175873.528712:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175873.528714:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175873.528728:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175873.528731:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175874.552693:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175874.552704:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175874.552707:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175874.552715:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175874.552722:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175874.552728:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175874.552733:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175874.552738:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175874.552742:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175875.128814:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175875.128819:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175875.128823:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175875.576752:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175875.576764:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175875.576768:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175875.576780:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175875.576790:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175875.576797:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175875.576806:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175875.576815:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175875.576823:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175875.578256:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175875.578267:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:3.1:1786175875.579208:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175875.579212:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:3.1:1786175875.579622:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175875.579626:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:3.1:1786175875.579956:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175875.579958:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:3.1:1786175875.580231:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175875.580235:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 57 secs later)
00000100:00000001:2.0:1786175876.600658:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175876.600667:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175876.600669:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175876.600675:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175876.600679:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175876.600683:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175876.600686:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175876.600688:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175876.600691:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175877.624716:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175877.624729:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175877.624732:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175877.624741:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175877.624746:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175877.624752:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175877.624756:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175877.624760:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175877.624764:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175878.200818:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175878.200830:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175878.200839:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175878.648660:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175878.648669:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175878.648672:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175878.648679:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175878.648684:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175878.648690:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175878.648694:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175878.648698:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175878.648702:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175879.672621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175879.672633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175879.672636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175879.672644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175879.672649:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175879.672655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175879.672659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175879.672662:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175879.672666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175880.696663:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175880.696680:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175880.696687:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175880.696695:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175880.696702:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175880.696714:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6104
00000400:00000200:2.0:1786175880.696727:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175880.696738:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175880.696742:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175880.696747:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.696750:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175880.696753:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.696755:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175880.696759:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175880.696761:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175880.696765:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175880.696768:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175880.696770:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175880.696773:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175880.696776:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.696778:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175880.696780:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.696782:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175880.696784:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175880.696787:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175880.696789:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175880.696792:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175880.696796:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175880.696799:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.696801:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175880.696803:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175880.696814:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786175880.696818:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175880.696830:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175880.696832:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.696836:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175880.696840:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175880.696845:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175880.696854:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000001:2.0:1786175880.696862:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175880.696872:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856184192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175880.696883:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175880.696888:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175880.696893:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856184192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175880.696913:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175880.696916:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175880.696919:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175880.696921:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175880.696923:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175880.696926:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175880.696928:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6104
00000400:00000200:2.0:1786175880.696933:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175880.696937:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175880.696939:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175880.696946:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.696950:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175880.696954:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.696959:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175880.696963:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175880.696965:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175880.696967:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175880.696970:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175880.696973:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175880.696983:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175880.696996:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697007:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175880.697013:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697018:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175880.697024:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175880.697037:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697041:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175880.697045:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175880.697055:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175880.697059:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.697067:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175880.697074:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175880.697080:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.697083:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175880.697086:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786175880.697091:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175880.697106:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175880.697109:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.697119:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175880.697129:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175880.697133:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175880.697148:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786175880.697158:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175880.697170:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856184320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175880.697193:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175880.697203:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175880.697218:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856184320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175880.697256:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175880.697266:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175880.697277:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175880.697280:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175880.697282:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175880.697286:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175880.697293:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6104
00000400:00000200:2.0:1786175880.697332:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175880.697339:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175880.697346:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175880.697359:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697377:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175880.697390:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697401:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.1:1786175880.697404:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175880.697415:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000020:00000001:2.0:1786175880.697416:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175880.697422:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175880.697427:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175880.697437:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175880.697447:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175880.697452:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175880.697460:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697472:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175880.697477:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697499:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175880.697504:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175880.697508:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697537:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175880.697539:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175880.697543:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175880.697546:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.697552:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175880.697555:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175880.697562:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.697567:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175880.697579:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786175880.697584:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175880.697590:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175880.697595:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.697603:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175880.697607:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175880.697611:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175880.697621:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786175880.697627:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175880.697636:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856184448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175880.697652:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175880.697656:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175880.697665:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856184448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175880.697675:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175880.697680:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175880.697686:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175880.697689:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175880.697691:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175880.697696:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175880.697701:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6104
00000400:00000200:2.0:1786175880.697711:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175880.697718:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175880.697724:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175880.697735:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697744:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175880.697747:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697754:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175880.697765:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175880.697769:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175880.697777:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175880.697779:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175880.697782:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175880.697788:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175880.697803:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697809:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175880.697822:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697829:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175880.697833:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175880.697843:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175880.697851:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175880.697858:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175880.697866:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175880.697874:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.697878:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175880.697884:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175880.697895:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.697899:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175880.697906:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786175880.697912:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175880.697920:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175880.697926:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.697935:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175880.697939:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175880.697946:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175880.697961:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786175880.697973:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175880.697987:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856184576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175880.698032:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175880.698038:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175880.698047:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856184576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175880.698079:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175880.698086:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175880.698096:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175880.698098:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175880.698101:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175880.698116:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175880.698125:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6104
00000400:00000200:2.0:1786175880.698138:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175880.698147:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175880.698156:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175880.698166:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.698179:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175880.698189:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.698198:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175880.698208:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175880.698211:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175880.698218:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175880.698229:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175880.698244:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175880.698254:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175880.698269:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.698274:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175880.698282:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175880.698286:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175880.698294:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175880.698311:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175880.698316:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175880.698326:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175880.698331:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175880.698338:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.698348:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175880.698355:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175880.698372:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.698382:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175880.698397:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786175880.698410:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175880.698424:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175880.698427:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175880.698431:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175880.698436:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175880.698443:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175880.698456:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786175880.698464:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175880.698474:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856184704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175880.698507:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175880.698535:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175880.698542:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856184704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175880.698614:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175880.698627:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6114)
00000800:00000200:3.1:1786175880.698687:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175880.698689:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786175880.698882:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175880.698884:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786175880.699145:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175880.699146:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1786175880.699405:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175880.699406:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000020:00000001:2.0:1786175881.272685:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175881.272691:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175881.272695:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175884.344703:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175884.344707:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175884.344710:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786175885.816647:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175885.816665:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175885.816672:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175885.816681:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175885.816691:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175885.816700:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6109
00000400:00000200:2.0:1786175885.816709:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175885.816715:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175885.816718:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175885.816722:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.816726:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175885.816727:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.816729:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175885.816733:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175885.816735:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175885.816737:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175885.816740:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175885.816742:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175885.816744:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175885.816746:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.816747:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175885.816749:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.816750:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175885.816752:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175885.816754:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175885.816755:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175885.816758:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175885.816760:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175885.816762:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.816765:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175885.816766:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175885.816774:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786175885.816777:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175885.816780:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175885.816781:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.816783:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175885.816785:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175885.816789:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175885.816795:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786175885.816801:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175885.816808:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856184832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175885.816816:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175885.816820:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175885.816823:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856184832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175885.816843:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175885.816846:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175885.816852:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175885.816853:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175885.816855:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175885.816857:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175885.816859:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6109
00000400:00000200:2.0:1786175885.816861:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175885.816863:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175885.816865:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175885.816867:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.816869:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175885.816871:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.816872:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175885.816874:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175885.816875:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175885.816877:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175885.816878:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175885.816879:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175885.816881:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175885.816883:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.816884:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175885.816886:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.816887:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175885.816888:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175885.816891:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175885.816894:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175885.816896:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175885.816901:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175885.816904:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.816907:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175885.816910:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175885.816914:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.816915:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175885.816920:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000020:00000040:2.0:1786175885.816924:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175885.816928:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175885.816930:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.816933:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175885.816935:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175885.816939:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175885.816942:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786175885.816946:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175885.816953:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856184960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175885.816966:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175885.816970:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175885.816977:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856184960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175885.816987:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175885.816992:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175885.816995:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175885.816998:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175885.817001:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175885.817004:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175885.817008:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6109
00000400:00000200:2.0:1786175885.817014:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175885.817017:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175885.817020:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175885.817023:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817026:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175885.817028:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817030:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175885.817032:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175885.817035:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175885.817040:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175885.817042:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175885.817046:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175885.817048:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175885.817053:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817056:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175885.817060:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817064:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175885.817068:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175885.817072:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817075:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175885.817077:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175885.817081:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175885.817085:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.817089:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175885.817092:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175885.817095:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.817098:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175885.817102:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000020:00000040:2.0:1786175885.817104:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175885.817106:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175885.817108:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.817111:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175885.817112:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175885.817113:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175885.817117:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786175885.817122:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175885.817130:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856185088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175885.817155:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175885.817158:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175885.817163:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856185088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175885.817177:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175885.817180:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175885.817185:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175885.817187:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175885.817189:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175885.817194:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175885.817197:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6109
00000400:00000200:2.0:1786175885.817203:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175885.817208:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175885.817212:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175885.817216:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:3.1:1786175885.817216:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786175885.817222:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.1:1786175885.817223:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00000040:2.0:1786175885.817226:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817230:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175885.817234:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175885.817236:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175885.817239:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175885.817241:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175885.817243:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175885.817246:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175885.817252:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817255:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175885.817258:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817261:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175885.817264:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175885.817268:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817270:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175885.817273:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175885.817276:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175885.817280:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.817284:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175885.817286:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175885.817289:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.817291:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175885.817296:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786175885.817300:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175885.817304:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175885.817306:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.817309:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175885.817311:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175885.817313:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175885.817316:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786175885.817318:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175885.817321:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856185216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175885.817342:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175885.817344:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175885.817351:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856185216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175885.817359:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175885.817361:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175885.817367:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175885.817369:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175885.817370:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175885.817373:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175885.817378:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6109
00000400:00000200:2.0:1786175885.817384:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175885.817387:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175885.817391:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175885.817395:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817396:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175885.817398:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817403:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175885.817404:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175885.817406:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175885.817407:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175885.817409:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175885.817412:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175885.817416:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175885.817420:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817423:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175885.817426:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817429:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175885.817432:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175885.817453:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175885.817458:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175885.817462:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175885.817466:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175885.817472:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.817477:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175885.817480:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175885.817487:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.817492:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175885.817500:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786175885.817505:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175885.817538:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175885.817540:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175885.817543:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175885.817544:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175885.817547:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175885.817551:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786175885.817558:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175885.817569:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856185344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175885.817591:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175885.817596:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175885.817607:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856185344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175885.817623:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175885.817631:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6119)
00000800:00000200:3.1:1786175885.817801:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175885.817802:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175885.818017:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175885.818018:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175885.818196:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175885.818198:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786175885.818393:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175885.818394:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000020:00000001:2.0:1786175887.416724:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175887.416728:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175887.416732:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175890.488738:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175890.488746:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175890.488751:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786175890.936586:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175890.936594:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175890.936596:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175890.936602:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175890.936608:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175890.936615:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6114
00000400:00000200:2.0:1786175890.936625:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175890.936633:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175890.936636:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175890.936639:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936643:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175890.936645:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936647:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175890.936651:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175890.936652:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175890.936655:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175890.936657:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175890.936659:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175890.936660:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175890.936662:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936664:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175890.936665:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936666:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175890.936668:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175890.936670:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936672:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175890.936675:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175890.936677:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175890.936680:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.936681:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175890.936682:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175890.936689:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786175890.936692:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175890.936695:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175890.936697:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.936699:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175890.936701:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175890.936705:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175890.936711:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:2.0:1786175890.936716:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175890.936723:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856185472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175890.936731:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175890.936735:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175890.936739:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856185472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175890.936758:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175890.936761:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175890.936763:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175890.936764:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175890.936765:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175890.936767:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175890.936769:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6114
00000400:00000200:2.0:1786175890.936771:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175890.936773:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175890.936775:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175890.936777:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936778:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175890.936780:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936782:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175890.936783:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175890.936785:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175890.936786:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175890.936787:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175890.936789:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175890.936790:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175890.936792:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936795:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175890.936796:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936800:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175890.936802:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175890.936806:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936809:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175890.936812:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175890.936816:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175890.936821:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.936824:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175890.936830:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175890.936834:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.936836:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175890.936839:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786175890.936843:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175890.936847:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175890.936850:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.936852:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175890.936853:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175890.936855:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175890.936859:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786175890.936863:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175890.936871:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856185600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175890.936880:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175890.936883:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175890.936889:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856185600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175890.936897:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175890.936900:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175890.936907:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175890.936910:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175890.936912:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175890.936915:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175890.936920:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6114
00000400:00000200:2.0:1786175890.936926:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175890.936930:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175890.936932:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175890.936937:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936938:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175890.936940:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936945:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175890.936949:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175890.936958:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175890.936962:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175890.936965:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175890.936969:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175890.936971:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175890.936979:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936982:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175890.936986:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.936990:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175890.936993:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175890.936995:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175890.937000:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175890.937004:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175890.937007:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175890.937008:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.937010:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175890.937012:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175890.937017:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.937018:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175890.937021:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786175890.937025:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175890.937029:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175890.937031:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.937034:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175890.937036:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175890.937039:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175890.937043:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786175890.937049:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175890.937056:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856185728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175890.937068:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175890.937070:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175890.937078:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856185728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175890.937093:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175890.937098:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175890.937101:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175890.937105:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175890.937107:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175890.937109:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175890.937114:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6114
00000400:00000200:2.0:1786175890.937118:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175890.937124:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175890.937126:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175890.937130:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.937134:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175890.937136:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.937138:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175890.937142:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000200:3.1:1786175890.937142:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1786175890.937144:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175890.937146:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175890.937150:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000800:00000200:3.1:1786175890.937151:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:2.0:1786175890.937152:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175890.937156:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175890.937167:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.937169:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175890.937172:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.937176:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175890.937178:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175890.937181:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175890.937187:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175890.937188:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175890.937191:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175890.937194:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.937198:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175890.937200:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175890.937202:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.937207:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175890.937213:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786175890.937217:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175890.937220:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175890.937222:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.937227:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175890.937229:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175890.937232:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175890.937237:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786175890.937240:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175890.937248:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856185856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175890.937272:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175890.937274:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175890.937279:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856185856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175890.937293:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175890.937297:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175890.937302:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175890.937303:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175890.937304:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175890.937306:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175890.937308:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6114
00000400:00000200:2.0:1786175890.937312:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175890.937314:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175890.937316:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175890.937318:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.937323:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175890.937326:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.937327:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175890.937329:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175890.937329:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175890.937332:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175890.937335:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175890.937338:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175890.937343:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175890.937346:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.937350:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175890.937352:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175890.937357:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175890.937360:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175890.937364:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175890.937367:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175890.937370:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175890.937372:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175890.937376:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.937380:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175890.937382:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175890.937407:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.937410:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175890.937415:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786175890.937421:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175890.937422:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175890.937423:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175890.937425:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175890.937427:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175890.937429:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175890.937434:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786175890.937438:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175890.937444:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856185984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175890.937469:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175890.937474:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175890.937477:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856185984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175890.937486:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175890.937493:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6124)
00000800:00000200:3.1:1786175890.937731:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175890.937733:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175890.937923:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175890.937925:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175890.938121:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175890.938122:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786175890.938308:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175890.938310:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:2.0:1786175893.560665:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175893.560670:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175893.560673:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786175896.057664:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175896.057673:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175896.058682:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175896.058684:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175896.059057:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175896.059061:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175896.059382:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175896.059385:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175896.059795:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175896.059799:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:2.0:1786175896.632649:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175896.632652:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175896.632655:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175899.704659:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175899.704663:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175899.704666:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786175901.176607:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175901.176613:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175901.176616:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175901.176622:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175901.176626:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175901.176632:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6124
00000400:00000200:2.0:1786175901.176640:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175901.176647:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175901.176649:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175901.176652:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.176654:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175901.176656:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.176658:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175901.176661:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175901.176662:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175901.176664:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175901.176667:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175901.176668:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175901.176670:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175901.176672:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.176673:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175901.176685:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.176686:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175901.176692:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175901.176695:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175901.176697:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175901.176699:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175901.176701:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175901.176703:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.176705:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175901.176707:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175901.176715:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000020:00000040:2.0:1786175901.176718:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175901.176721:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175901.176722:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.176724:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175901.176726:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175901.176731:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175901.176736:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786175901.176742:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175901.176750:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856186752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175901.176757:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175901.176761:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175901.176764:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856186752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175901.176784:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175901.176787:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175901.176790:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175901.176792:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175901.176793:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175901.176796:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175901.176799:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6124
00000400:00000200:2.0:1786175901.176803:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175901.176807:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175901.176810:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175901.176815:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.176817:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175901.176821:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.176825:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175901.176831:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175901.176834:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175901.176839:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175901.176843:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175901.176847:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175901.176851:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175901.176858:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.176863:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175901.176868:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.176871:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175901.176874:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175901.176888:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175901.176890:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175901.176893:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175901.176898:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175901.176903:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.176905:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175901.176908:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175901.176915:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.176918:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175901.176923:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000020:00000040:2.0:1786175901.176927:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175901.176930:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175901.176931:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.176933:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175901.176936:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175901.176938:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175901.176943:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786175901.176949:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175901.176957:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856186880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175901.176974:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175901.176978:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175901.176985:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856186880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175901.177000:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175901.177003:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175901.177005:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175901.177008:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175901.177011:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175901.177014:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175901.177018:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6124
00000400:00000200:2.0:1786175901.177046:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175901.177054:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175901.177057:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175901.177065:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177068:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175901.177070:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177073:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175901.177075:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175901.177076:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175901.177080:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175901.177084:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175901.177089:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175901.177094:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:3.1:1786175901.177094:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175901.177101:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000040:2.0:1786175901.177102:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177107:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175901.177112:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177113:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175901.177114:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175901.177116:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177118:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175901.177119:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175901.177124:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175901.177128:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.177130:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175901.177133:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175901.177136:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.177151:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175901.177156:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000020:00000040:2.0:1786175901.177161:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175901.177164:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175901.177167:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.177170:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175901.177171:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175901.177174:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175901.177180:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786175901.177184:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175901.177191:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856187008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175901.177206:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175901.177210:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175901.177218:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856187008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175901.177228:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175901.177233:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175901.177238:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175901.177242:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175901.177243:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175901.177247:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175901.177249:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6124
00000400:00000200:2.0:1786175901.177255:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175901.177264:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175901.177266:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175901.177268:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177271:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175901.177275:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177282:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175901.177302:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175901.177305:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175901.177309:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175901.177313:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175901.177318:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175901.177322:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175901.177330:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177336:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175901.177339:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177343:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175901.177345:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175901.177347:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177348:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175901.177359:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175901.177362:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175901.177366:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.177369:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175901.177371:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175901.177375:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.177378:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175901.177383:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000020:00000040:2.0:1786175901.177388:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175901.177393:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175901.177397:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.177402:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175901.177404:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175901.177407:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175901.177414:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786175901.177417:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175901.177424:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856187136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175901.177437:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175901.177448:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175901.177457:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856187136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175901.177471:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175901.177476:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175901.177481:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175901.177483:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175901.177492:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175901.177497:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175901.177502:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6124
00000400:00000200:2.0:1786175901.177539:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175901.177547:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175901.177551:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175901.177558:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177563:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175901.177568:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177572:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175901.177577:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175901.177580:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175901.177583:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175901.177588:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175901.177591:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175901.177595:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175901.177600:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177602:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175901.177606:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177611:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175901.177615:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175901.177620:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175901.177624:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175901.177627:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175901.177629:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175901.177631:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.177634:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175901.177637:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175901.177642:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.177646:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175901.177651:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000020:00000040:2.0:1786175901.177656:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175901.177662:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175901.177665:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175901.177668:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175901.177670:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175901.177674:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175901.177680:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786175901.177687:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175901.177695:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856187264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175901.177710:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175901.177711:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175901.177714:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856187264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175901.177730:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175901.177736:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6134)
00000800:00000200:3.1:1786175901.177819:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175901.177821:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175901.178023:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175901.178024:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175901.178210:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175901.178212:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175901.178549:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175901.178551:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:2.0:1786175902.776687:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175902.776692:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175902.776705:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175905.848731:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175905.848734:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175905.848737:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786175906.297168:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175906.297172:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175906.297921:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175906.297922:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175906.298138:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175906.298140:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175906.298390:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175906.298392:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175906.298586:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175906.298587:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:2.0:1786175908.920757:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175908.920761:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175908.920763:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786175911.417485:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175911.417490:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175911.418371:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175911.418373:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175911.418731:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175911.418733:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175911.418965:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175911.418967:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175911.419259:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175911.419263:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:2.0:1786175911.992717:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175911.992722:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175911.992725:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175915.064751:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175915.064757:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175915.064762:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786175916.537214:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175916.537219:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175916.537966:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175916.537970:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175916.538293:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175916.538294:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175916.538537:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175916.538539:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175916.538728:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175916.538729:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:2.0:1786175918.136658:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175918.136662:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175918.136664:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175921.208746:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175921.208750:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175921.208753:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786175921.657179:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175921.657183:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1786175921.657874:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175921.657879:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1786175921.658164:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175921.658165:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1786175921.658399:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175921.658401:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1786175921.658668:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175921.658671:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000020:00000001:2.0:1786175924.280792:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175924.280799:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175924.280804:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786175926.777598:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175926.777613:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786175926.778216:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175926.778218:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786175926.778466:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175926.778469:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786175926.778747:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175926.778749:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786175926.779010:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175926.779011:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:2.0:1786175927.352752:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175927.352756:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175927.352759:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175930.424693:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175930.424697:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175930.424700:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175931.896670:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175931.896681:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175931.896688:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175931.896697:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175931.896705:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175931.896711:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175931.896716:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175931.896721:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175931.896727:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175931.897583:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175931.897593:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786175931.898657:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175931.898660:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786175931.899014:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175931.899025:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786175931.899299:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175931.899301:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786175931.899804:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175931.899807:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:2.0:1786175932.920627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175932.920638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175932.920641:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175932.920645:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175932.920650:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175932.920653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175932.920656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175932.920658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175932.920661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175933.496701:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175933.496704:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175933.496707:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175933.944642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175933.944648:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175933.944650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175933.944655:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175933.944658:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175933.944660:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175933.944662:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175933.944665:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175933.944667:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175934.968627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175934.968634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175934.968636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175934.968640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175934.968643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175934.968645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175934.968648:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175934.968650:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175934.968652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175935.992583:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175935.992589:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175935.992591:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175935.992595:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175935.992599:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175935.992602:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175935.992604:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175935.992606:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175935.992608:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175936.568716:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175936.568720:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175936.568723:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175937.016624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175937.016629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175937.016632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175937.016638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175937.016643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175937.016648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175937.016651:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175937.016655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175937.016659:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175937.017128:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175937.017348:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175937.017507:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175937.017717:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175937.017872:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786175938.040725:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175938.040734:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175938.040737:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175938.040743:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175938.040748:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175938.040752:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175938.040756:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175938.040759:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175938.040763:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175939.064694:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175939.064703:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175939.064706:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175939.064713:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175939.064717:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175939.064722:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175939.064726:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175939.064730:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175939.064733:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175939.640683:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175939.640688:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175939.640691:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175940.088938:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175940.088945:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175940.088947:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175940.088952:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175940.088956:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175940.088959:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175940.088962:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175940.088965:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175940.088968:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175941.112621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175941.112640:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175941.112641:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175941.112646:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175941.112649:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175941.112651:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175941.112654:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175941.112656:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175941.112658:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786175942.137377:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175942.137385:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786175942.138155:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175942.138157:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786175942.138473:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175942.138474:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786175942.138745:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175942.138747:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786175942.138983:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175942.138985:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:2.0:1786175942.712670:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175942.712685:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175942.712689:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175945.784667:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175945.784672:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175945.784676:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786175947.257892:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175947.257907:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786175947.259271:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175947.259290:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786175947.259717:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175947.259720:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786175947.260121:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175947.260125:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786175947.260487:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175947.260489:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:2.0:1786175948.856953:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175948.856961:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175948.856967:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786175951.928875:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175951.928881:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175951.928886:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786175952.376721:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175952.376737:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175952.376741:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175952.376747:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175952.376754:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175952.376766:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6175
00000400:00000200:2.0:1786175952.376780:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175952.376805:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175952.376811:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175952.376815:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.376818:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175952.376821:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.376840:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175952.376846:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175952.376849:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175952.376852:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175952.376858:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175952.376860:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175952.376863:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175952.376867:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.376868:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175952.376870:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.376872:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175952.376875:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175952.376878:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175952.376881:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175952.376884:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175952.376889:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175952.376892:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.376894:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175952.376897:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175952.376912:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786175952.376916:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175952.376921:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175952.376923:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.376928:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175952.376932:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175952.376938:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175952.376976:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786175952.376985:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175952.376997:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856193152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175952.377009:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175952.377014:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175952.377019:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856193152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175952.377047:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175952.377060:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175952.377064:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175952.377066:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175952.377068:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175952.377071:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175952.377074:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6175
00000400:00000200:2.0:1786175952.377078:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175952.377081:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175952.377084:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175952.377087:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.377090:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175952.377094:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.377098:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175952.377102:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175952.377105:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175952.377108:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175952.377112:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175952.377115:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175952.377119:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175952.377125:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.377136:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175952.377152:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.377160:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175952.377171:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175952.377186:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175952.377195:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175952.377204:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175952.377231:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175952.377242:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.377255:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175952.377287:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175952.377297:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.377305:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175952.377322:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786175952.377330:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175952.377334:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175952.377336:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.377339:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175952.377342:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175952.377348:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175952.377356:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786175952.377400:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175952.377423:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856193280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175952.377446:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175952.377454:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175952.377472:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856193280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175952.377609:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175952.377615:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175952.377620:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175952.377632:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175952.377635:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175952.377645:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175952.377655:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6175
00000400:00000200:2.0:1786175952.377684:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175952.377697:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175952.377724:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175952.377740:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.377753:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175952.377766:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.377783:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175952.377810:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175952.377813:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175952.377817:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175952.377842:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175952.377853:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175952.377861:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175952.377872:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:0.1:1786175952.377874:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175952.377889:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1786175952.377893:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175952.377903:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.377915:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175952.377928:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175952.377936:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175952.377944:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175952.377972:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175952.377978:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175952.377983:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.377988:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175952.377995:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175952.378000:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.378005:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175952.378010:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786175952.378015:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175952.378022:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175952.378026:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.378037:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175952.378041:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175952.378044:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175952.378055:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786175952.378061:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175952.378076:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856193408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175952.378166:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175952.378174:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175952.378186:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856193408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175952.378203:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175952.378208:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175952.378242:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175952.378253:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175952.378263:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175952.378276:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175952.378314:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6175
00000400:00000200:2.0:1786175952.378333:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175952.378342:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175952.378364:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175952.378371:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.378380:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175952.378395:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.378398:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175952.378410:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175952.378423:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175952.378426:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175952.378433:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175952.378445:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175952.378457:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175952.378470:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.378486:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175952.378523:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.378537:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175952.378546:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175952.378552:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175952.378555:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175952.378557:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175952.378561:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175952.378592:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.378595:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175952.378603:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175952.378642:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.378650:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175952.378661:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786175952.378668:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175952.378679:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175952.378684:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.378689:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175952.378691:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175952.378719:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175952.378732:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786175952.378748:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175952.378770:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856193536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175952.378815:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175952.378818:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175952.378856:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856193536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175952.378894:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175952.378907:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175952.378918:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175952.378926:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175952.378930:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175952.378975:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175952.378988:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6175
00000400:00000200:2.0:1786175952.378994:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175952.379000:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175952.379005:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175952.379013:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.379017:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175952.379026:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.379036:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175952.379040:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175952.379046:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175952.379047:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175952.379051:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175952.379053:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175952.379067:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175952.379077:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.379088:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175952.379100:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175952.379113:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175952.379128:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175952.379149:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175952.379160:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175952.379167:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175952.379173:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175952.379192:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.379211:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175952.379216:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175952.379229:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.379239:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175952.379255:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786175952.379264:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175952.379280:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175952.379286:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175952.379291:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175952.379294:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175952.379298:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175952.379310:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:2.0:1786175952.379317:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175952.379337:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856193664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175952.379376:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175952.379386:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175952.379403:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856193664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175952.379450:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175952.379471:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6185)
00000800:00000200:0.1:1786175952.379525:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175952.379529:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786175952.379926:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175952.379928:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786175952.380276:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175952.380278:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786175952.380845:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175952.380848:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:2.0:1786175955.000830:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175955.000840:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175955.000847:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786175957.496644:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175957.496653:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175957.496656:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175957.496673:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175957.496677:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175957.496686:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6180
00000400:00000200:2.0:1786175957.496696:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175957.496703:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175957.496707:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175957.496709:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.496713:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175957.496715:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.496718:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175957.496722:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175957.496725:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175957.496728:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175957.496758:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175957.496761:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175957.496763:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175957.496767:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.496768:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175957.496770:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.496771:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175957.496774:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175957.496777:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175957.496778:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175957.496781:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175957.496785:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175957.496793:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.496807:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175957.496809:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175957.496821:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786175957.496824:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175957.496829:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175957.496830:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.496834:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175957.496837:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175957.496842:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175957.496851:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786175957.496859:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175957.496882:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856193792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175957.496892:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175957.496897:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175957.496902:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856193792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175957.496930:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175957.496934:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175957.496937:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175957.496938:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175957.496940:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175957.496942:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175957.496944:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6180
00000400:00000200:2.0:1786175957.496948:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175957.496950:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175957.496952:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175957.496955:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.496957:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175957.496958:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.496960:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175957.496962:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175957.496964:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175957.496965:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175957.496967:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175957.496970:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175957.496972:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175957.496974:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.496976:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175957.496977:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.496979:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175957.496981:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175957.496983:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175957.496986:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175957.496991:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175957.496998:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175957.497013:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.497016:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175957.497023:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175957.497030:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.497034:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175957.497039:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786175957.497044:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175957.497048:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175957.497051:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.497056:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175957.497057:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175957.497059:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175957.497062:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786175957.497066:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175957.497075:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856193920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175957.497087:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175957.497091:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175957.497097:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856193920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175957.497107:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175957.497111:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175957.497114:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175957.497118:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175957.497121:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175957.497124:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175957.497129:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6180
00000400:00000200:2.0:1786175957.497132:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175957.497136:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175957.497139:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175957.497143:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497145:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175957.497150:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497154:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175957.497159:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175957.497162:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175957.497167:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175957.497171:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175957.497174:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175957.497181:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175957.497191:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497199:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175957.497207:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497215:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175957.497221:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175957.497229:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497232:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175957.497237:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175957.497242:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175957.497245:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.497248:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175957.497254:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175957.497259:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.497262:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175957.497276:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000020:00000040:2.0:1786175957.497281:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175957.497289:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175957.497291:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.497295:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175957.497297:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175957.497298:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175957.497302:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786175957.497318:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175957.497327:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856194048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175957.497344:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175957.497349:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175957.497354:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856194048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175957.497361:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175957.497364:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175957.497381:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175957.497383:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175957.497384:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175957.497389:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175957.497394:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6180
00000400:00000200:2.0:1786175957.497401:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175957.497409:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:0.1:1786175957.497409:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786175957.497415:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:0.1:1786175957.497417:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:2.0:1786175957.497420:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497424:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175957.497431:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497434:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175957.497448:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175957.497449:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175957.497451:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175957.497454:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175957.497459:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175957.497464:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175957.497469:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497471:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175957.497476:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497480:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175957.497483:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175957.497491:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497498:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175957.497503:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175957.497506:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175957.497569:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.497575:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175957.497580:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175957.497587:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.497595:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175957.497604:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000020:00000040:2.0:1786175957.497608:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175957.497612:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175957.497616:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.497621:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175957.497627:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175957.497631:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175957.497642:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786175957.497651:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175957.497659:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856194176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175957.497669:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175957.497676:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175957.497681:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856194176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175957.497709:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175957.497720:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175957.497729:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175957.497731:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175957.497737:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175957.497739:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175957.497766:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6180
00000400:00000200:2.0:1786175957.497773:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175957.497781:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175957.497810:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175957.497815:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497819:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175957.497823:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497828:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175957.497834:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175957.497836:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175957.497839:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175957.497844:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175957.497848:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175957.497853:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175957.497859:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497866:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175957.497870:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497873:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175957.497877:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175957.497884:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175957.497889:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175957.497894:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175957.497898:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175957.497903:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.497909:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175957.497912:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175957.497916:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.497919:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175957.497926:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786175957.497931:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175957.497937:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175957.497940:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175957.497945:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175957.497949:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175957.497952:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175957.497958:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786175957.497965:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175957.497975:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856194304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175957.498001:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175957.498007:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175957.498014:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856194304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175957.498032:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175957.498039:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6190)
00000800:00000200:0.1:1786175957.498222:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175957.498224:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786175957.498511:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175957.498513:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786175957.498827:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175957.498830:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786175957.499048:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786175957.499050:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:2.0:1786175958.072727:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175958.072732:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175958.072752:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175959.544623:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175959.544633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175959.544635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175959.544641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175959.544645:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175959.544649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175959.544653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175959.544656:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175959.544659:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175960.568650:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175960.568662:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175960.568665:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175960.568673:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175960.568679:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175960.568684:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175960.568689:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175960.568693:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175960.568698:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175961.144696:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175961.144702:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175961.144731:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175961.592719:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175961.592732:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175961.592735:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175961.592743:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175961.592749:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175961.592754:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175961.592758:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175961.592762:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175961.592766:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175962.616777:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175962.616790:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175962.616797:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175962.616814:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175962.616824:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175962.616835:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175962.616840:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175962.616843:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175962.616847:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175962.617722:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175962.617729:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175962.619059:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175962.619068:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175962.619622:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175962.619625:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175962.620109:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175962.620113:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786175962.620455:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175962.620458:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:2.0:1786175963.640652:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175963.640660:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175963.640662:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175963.640667:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175963.640671:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175963.640675:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175963.640679:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175963.640701:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175963.640705:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175964.216844:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175964.216849:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175964.216853:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175964.664779:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175964.664792:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175964.664796:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175964.664806:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175964.664811:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175964.664817:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175964.664822:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175964.664826:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175964.664830:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175965.688744:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175965.688756:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175965.688760:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175965.688779:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175965.688785:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175965.688790:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175965.688794:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175965.688797:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175965.688800:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175966.712733:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175966.712743:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175966.712745:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175966.712752:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175966.712756:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175966.712761:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175966.712764:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175966.712767:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175966.712770:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175967.288888:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175967.288899:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175967.288907:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175967.736670:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175967.736678:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175967.736680:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175967.736687:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175967.736691:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175967.736695:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175967.736702:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175967.736706:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175967.736711:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175967.737324:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175967.737332:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175967.737919:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175967.737921:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175967.738153:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175967.738155:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175967.738357:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175967.738359:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786175967.738570:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175967.738572:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1786175968.760600:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175968.760609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175968.760611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175968.760617:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175968.760621:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175968.760625:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175968.760627:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175968.760630:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175968.760656:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175969.784654:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175969.784677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175969.784681:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175969.784690:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175969.784696:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175969.784701:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175969.784705:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175969.784710:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175969.784714:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175970.360934:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175970.360942:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175970.360950:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175970.808605:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175970.808640:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175970.808642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175970.808649:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175970.808653:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175970.808657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175970.808660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175970.808662:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175970.808665:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175971.832731:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175971.832743:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175971.832746:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175971.832755:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175971.832760:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175971.832765:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175971.832768:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175971.832772:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175971.832775:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175972.856732:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175972.856745:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175972.856751:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175972.856765:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175972.856778:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175972.856784:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175972.856789:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175972.856797:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175972.856803:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175972.857712:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175972.857718:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175972.858837:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175972.858840:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175972.859167:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175972.859169:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175972.859504:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175972.859506:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786175972.860006:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175972.860009:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:2.0:1786175973.432834:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175973.432842:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175973.432847:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175973.880770:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175973.880786:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175973.880791:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175973.880803:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175973.880811:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175973.880819:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175973.880826:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175973.880831:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175973.880838:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175974.904623:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175974.904633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175974.904635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175974.904641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175974.904645:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175974.904649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175974.904652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175974.904655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175974.904658:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175975.928673:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175975.928683:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175975.928686:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175975.928693:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175975.928697:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175975.928702:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175975.928705:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175975.928708:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175975.928712:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175976.504834:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175976.504844:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175976.504850:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175976.952710:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175976.952721:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175976.952724:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175976.952731:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175976.952735:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175976.952740:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175976.952743:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175976.952747:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175976.952750:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175977.976668:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175977.976678:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175977.976681:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175977.976689:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175977.976695:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175977.976700:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175977.976705:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175977.976709:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175977.976712:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175977.977524:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175977.977534:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175977.978440:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175977.978443:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175977.978844:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175977.978848:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175977.979146:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175977.979148:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786175977.979437:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175977.979440:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1786175979.000740:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175979.000754:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175979.000758:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175979.000768:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175979.000775:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175979.000781:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175979.000786:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175979.000791:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175979.000796:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175979.576880:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175979.576892:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175979.576901:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175980.024656:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175980.024677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175980.024680:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175980.024686:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175980.024692:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175980.024697:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175980.024701:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175980.024704:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175980.024708:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175981.048613:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175981.048622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175981.048625:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175981.048631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175981.048636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175981.048640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175981.048644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175981.048647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175981.048651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175981.624798:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175981.624804:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175981.624809:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175982.072660:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175982.072672:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175982.072675:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175982.072685:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175982.072692:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175982.072698:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175982.072703:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175982.072708:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175982.072713:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175982.648843:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175982.648849:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175982.648854:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175983.096613:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175983.096624:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175983.096627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175983.096635:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175983.096641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175983.096645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175983.096650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175983.096654:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175983.096658:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175983.097791:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175983.097805:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175983.099052:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175983.099055:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175983.099432:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175983.099435:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175983.099907:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175983.099911:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786175983.100312:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175983.100315:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1786175984.120576:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175984.120582:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175984.120583:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175984.120588:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175984.120590:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175984.120593:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175984.120595:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175984.120597:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175984.120599:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175984.696737:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175984.696743:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175984.696748:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175985.144598:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175985.144607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175985.144610:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175985.144616:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175985.144621:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175985.144626:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175985.144629:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175985.144632:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175985.144636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175985.720820:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175985.720828:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175985.720835:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175986.168605:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175986.168614:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175986.168617:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175986.168624:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175986.168629:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175986.168632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175986.168636:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175986.168639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175986.168643:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175987.192613:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175987.192622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175987.192625:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175987.192650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175987.192656:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175987.192663:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175987.192668:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175987.192674:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175987.192680:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175987.768664:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175987.768667:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175987.768670:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175988.216637:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175988.216645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175988.216648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175988.216654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175988.216660:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175988.216664:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175988.216672:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175988.216676:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175988.216682:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175988.217757:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175988.217770:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175988.218730:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175988.218735:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175988.219140:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175988.219143:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175988.219668:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175988.219671:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786175988.220105:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175988.220109:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:2.0:1786175988.792782:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175988.792789:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175988.792794:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175989.240674:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175989.240703:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175989.240708:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175989.240718:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175989.240726:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175989.240733:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175989.240740:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175989.240744:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175989.240749:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175990.264681:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175990.264693:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175990.264697:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175990.264707:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175990.264712:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175990.264716:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175990.264720:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175990.264724:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175990.264728:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175990.840855:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175990.840863:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175990.840868:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175991.288694:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175991.288705:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175991.288708:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175991.288717:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175991.288722:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175991.288728:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175991.288732:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175991.288736:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175991.288740:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175991.864800:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175991.864806:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175991.864811:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175992.312643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175992.312653:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175992.312656:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175992.312665:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175992.312669:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175992.312674:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175992.312679:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175992.312683:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175992.312687:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786175993.208729:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786175993.208741:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786175993.208750:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:2.0:1786175993.336768:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175993.336780:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175993.336785:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175993.336797:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175993.336808:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175993.336822:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175993.336830:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175993.336836:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175993.336841:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175993.337509:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175993.337538:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175993.338272:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175993.338275:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175993.338625:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175993.338627:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175993.338913:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175993.338918:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786175993.339214:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175993.339216:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:2.0:1786175993.912749:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175993.912757:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175993.912765:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175994.360595:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175994.360602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175994.360604:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175994.360610:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175994.360614:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175994.360618:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175994.360620:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175994.360623:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175994.360626:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175994.936764:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175994.936770:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175994.936775:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175995.384699:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175995.384712:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175995.384716:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175995.384725:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175995.384731:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175995.384738:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175995.384742:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175995.384747:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175995.384751:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175996.408640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175996.408653:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175996.408658:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175996.408667:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175996.408673:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175996.408678:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175996.408683:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175996.408688:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175996.408694:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175996.984883:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175996.984893:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175996.984899:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786175997.432668:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786175997.432679:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786175997.432682:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175997.432689:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786175997.432695:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786175997.432700:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175997.432704:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786175997.432707:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786175997.432711:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786175998.008820:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786175998.008825:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786175998.008830:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786175998.456662:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175998.456680:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786175998.456685:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175998.456693:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175998.456701:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175998.456713:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6221
00000400:00000200:2.0:1786175998.456729:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175998.456744:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175998.456754:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175998.456767:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.456776:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175998.456783:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.456787:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175998.456793:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175998.456797:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786175998.456804:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175998.456810:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786175998.456817:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786175998.456822:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786175998.456828:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.456833:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175998.456837:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.456841:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175998.456847:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786175998.456854:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175998.456860:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175998.456865:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175998.456871:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175998.456876:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.456881:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786175998.456886:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175998.456899:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786175998.456904:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786175998.456909:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175998.456912:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.456915:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175998.456919:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175998.456926:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175998.456937:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786175998.456945:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175998.456958:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856198912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175998.456972:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175998.456978:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175998.456984:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856198912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175998.457017:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175998.457021:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175998.457026:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786175998.457028:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175998.457030:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175998.457034:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175998.457038:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6221
00000400:00000200:2.0:1786175998.457045:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175998.457052:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175998.457056:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175998.457062:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.457066:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175998.457073:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.457079:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175998.457085:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175998.457087:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786175998.457091:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175998.457095:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786175998.457105:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786175998.457115:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786175998.457128:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.457135:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175998.457142:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.457149:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175998.457161:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786175998.457168:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175998.457171:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175998.457173:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175998.457180:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175998.457186:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.457195:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175998.457202:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175998.457209:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.457216:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175998.457242:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000020:00000040:2.0:1786175998.457253:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175998.457262:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175998.457267:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.457270:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175998.457276:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175998.457279:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175998.457288:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786175998.457301:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175998.457318:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856199040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175998.457381:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175998.457390:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175998.457403:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856199040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175998.457428:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175998.457436:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175998.457446:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786175998.457451:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175998.457455:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175998.457467:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175998.457480:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6221
00000400:00000200:2.0:1786175998.457497:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175998.457562:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175998.457567:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175998.457574:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.457581:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175998.457594:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.457608:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.1:1786175998.457614:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786175998.457621:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000200:3.1:1786175998.457623:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000040:2.0:1786175998.457629:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786175998.457639:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175998.457650:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786175998.457665:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786175998.457674:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786175998.457686:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.457695:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175998.457703:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.457708:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175998.457715:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786175998.457727:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175998.457740:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175998.457748:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175998.457756:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175998.457760:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.457764:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175998.457767:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786175998.457771:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.457773:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175998.457779:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000020:00000040:2.0:1786175998.457783:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786175998.457791:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175998.457795:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.457807:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175998.457813:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175998.457821:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175998.457837:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786175998.457852:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175998.457873:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856199168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175998.457900:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175998.457908:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175998.457921:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856199168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175998.457946:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175998.457956:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175998.457968:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786175998.457975:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175998.457982:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175998.457990:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175998.458002:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6221
00000400:00000200:2.0:1786175998.458018:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175998.458033:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175998.458042:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175998.458055:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.458065:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175998.458072:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.458081:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175998.458090:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175998.458094:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786175998.458100:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175998.458107:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786175998.458115:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786175998.458128:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786175998.458139:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.458145:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175998.458155:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.458165:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175998.458174:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786175998.458184:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175998.458196:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175998.458201:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175998.458209:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175998.458222:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.458232:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175998.458240:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175998.458249:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.458256:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175998.458270:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786175998.458279:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175998.458290:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786175998.458295:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.458302:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175998.458308:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175998.458312:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175998.458325:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786175998.458335:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175998.458381:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856199296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175998.458419:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175998.458427:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175998.458440:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856199296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175998.458468:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786175998.458474:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786175998.458486:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786175998.458491:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786175998.458497:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786175998.458544:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786175998.458556:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6221
00000400:00000200:2.0:1786175998.458573:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786175998.458589:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786175998.458598:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786175998.458616:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.458627:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175998.458642:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.458654:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786175998.458663:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786175998.458672:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786175998.458682:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786175998.458692:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786175998.458702:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786175998.458712:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786175998.458725:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.458737:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786175998.458750:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786175998.458758:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786175998.458769:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786175998.458782:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786175998.458791:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786175998.458804:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786175998.458815:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786175998.458830:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.458839:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786175998.458849:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786175998.458864:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786175998.458876:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786175998.458892:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786175998.458903:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786175998.458917:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:3.1:1786175998.458920:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1786175998.458923:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786175998.458926:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1786175998.458929:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786175998.458935:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786175998.458942:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786175998.458952:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786175998.458969:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786175998.458985:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856199424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786175998.459012:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786175998.459020:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175998.459034:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856199424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786175998.459099:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786175998.459110:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6231)
00000800:00000200:3.1:1786175998.459443:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175998.459446:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175998.459873:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175998.459876:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786175998.460387:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786175998.460391:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:2.0:1786176000.056778:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176000.056787:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176000.056794:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176001.080790:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176001.080796:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176001.080801:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176003.128732:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176003.128737:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176003.128740:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176003.576673:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176003.576677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176003.576691:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176003.576695:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176003.576700:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176003.576704:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176003.576708:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176003.576710:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176003.576713:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176003.577206:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176003.577754:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176003.577971:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176003.578135:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176003.578545:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786176004.152679:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176004.152683:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176004.152686:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176004.600589:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176004.600595:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176004.600597:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176004.600602:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176004.600605:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176004.600608:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176004.600611:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176004.600613:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176004.600615:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176005.624577:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176005.624583:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176005.624585:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176005.624590:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176005.624593:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176005.624596:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176005.624598:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176005.624600:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176005.624602:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176006.200690:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176006.200693:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176006.200696:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176006.648651:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176006.648661:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176006.648663:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176006.648670:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176006.648675:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176006.648680:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176006.648685:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176006.648688:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176006.648692:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176007.224753:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176007.224758:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176007.224763:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176007.672649:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176007.672657:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176007.672661:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176007.672667:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176007.672672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176007.672677:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176007.672681:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176007.672684:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176007.672706:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176008.696647:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176008.696656:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176008.696660:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176008.696667:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176008.696672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176008.696677:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176008.696681:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176008.696687:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176008.696696:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176008.697894:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176008.697909:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176008.698872:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176008.698878:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176008.700077:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176008.700081:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176008.701052:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176008.701057:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176008.701548:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176008.701551:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:2.0:1786176009.272743:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176009.272751:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176009.272755:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176009.720629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176009.720637:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176009.720639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176009.720646:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176009.720651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176009.720655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176009.720658:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176009.720661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176009.720664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176010.296916:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176010.296923:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176010.296928:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176010.744637:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176010.744646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176010.744647:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176010.744654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176010.744658:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176010.744662:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176010.744664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176010.744667:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176010.744669:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176011.768617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176011.768628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176011.768631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176011.768640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176011.768647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176011.768653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176011.768658:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176011.768663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176011.768668:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176012.344711:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176012.344715:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176012.344719:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176012.792700:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176012.792708:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176012.792710:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176012.792716:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176012.792719:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176012.792724:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176012.792727:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176012.792729:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176012.792732:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176013.368741:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176013.368745:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176013.368749:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176013.816720:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176013.816728:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176013.816734:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176013.816746:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176013.816757:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176013.816781:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176013.816786:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176013.816790:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176013.816799:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176013.817829:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176013.817843:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176013.819100:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176013.819103:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176013.819420:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176013.819422:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176013.819817:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176013.819819:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176013.820110:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176013.820112:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1786176014.840617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176014.840626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176014.840630:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176014.840637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176014.840641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176014.840647:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176014.840650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176014.840654:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176014.840657:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176015.416789:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176015.416796:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176015.416801:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176015.864621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176015.864629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176015.864632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176015.864638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176015.864643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176015.864648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176015.864651:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176015.864655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176015.864658:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176016.440670:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176016.440674:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176016.440677:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176016.888607:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176016.888614:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176016.888616:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176016.888621:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176016.888624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176016.888627:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176016.888628:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176016.888631:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176016.888633:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176017.464790:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176017.464795:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176017.464799:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176017.912612:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176017.912620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176017.912623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176017.912629:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176017.912633:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176017.912638:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176017.912641:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176017.912644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176017.912647:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176018.488727:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176018.488730:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176018.488733:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176018.936644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176018.936650:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176018.936656:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176018.936663:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176018.936669:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176018.936675:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176018.936698:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176018.936702:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176018.936704:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176018.937472:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176018.937478:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176018.938323:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176018.938325:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176018.938630:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176018.938632:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176018.938888:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176018.938890:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176018.939212:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176018.939214:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:2.0:1786176019.512838:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176019.512843:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176019.512847:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176019.960614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176019.960621:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176019.960622:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176019.960627:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176019.960630:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176019.960633:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176019.960635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176019.960638:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176019.960640:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176020.536677:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176020.536681:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176020.536683:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176020.984638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176020.984646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176020.984648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176020.984654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176020.984658:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176020.984661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176020.984664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176020.984667:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176020.984670:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176021.560797:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176021.560805:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176021.560813:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176022.008647:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176022.008653:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176022.008655:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176022.008660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176022.008664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176022.008667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176022.008670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176022.008673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176022.008675:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176022.584753:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176022.584758:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176022.584762:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176023.032720:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176023.032737:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176023.032742:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176023.032754:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176023.032761:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176023.032769:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176023.032776:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176023.032781:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176023.032788:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176023.608778:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176023.608787:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176023.608793:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176024.056730:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176024.056747:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176024.056758:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176024.056771:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176024.056787:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176024.056799:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176024.056812:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176024.056825:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176024.056836:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176024.057319:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176024.057325:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176024.058086:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176024.058089:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176024.058397:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176024.058399:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176024.058652:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176024.058654:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176024.058962:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176024.058964:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:2.0:1786176024.632719:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176024.632726:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176024.632731:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176025.080616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176025.080624:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176025.080627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176025.080633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176025.080637:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176025.080642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176025.080645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176025.080648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176025.080651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176025.656653:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176025.656656:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176025.656659:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176026.104620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176026.104626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176026.104629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176026.104634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176026.104638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176026.104642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176026.104645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176026.104648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176026.104650:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176026.680706:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176026.680711:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176026.680714:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176027.128584:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176027.128591:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176027.128593:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176027.128597:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176027.128600:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176027.128603:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176027.128605:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176027.128607:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176027.128609:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176027.704720:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176027.704725:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176027.704728:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176028.152660:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176028.152669:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176028.152671:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176028.152677:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176028.152681:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176028.152685:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176028.152688:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176028.152690:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176028.152693:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176028.728788:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176028.728791:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176028.728794:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176029.176644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176029.176669:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176029.176670:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176029.176675:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176029.176678:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176029.176681:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176029.176683:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176029.176686:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176029.176689:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176029.177306:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176029.177312:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176029.178141:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176029.178143:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176029.178330:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176029.178332:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176029.178583:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176029.178585:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176029.178831:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176029.178833:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:2.0:1786176029.752674:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176029.752677:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176029.752680:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176030.200603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176030.200611:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176030.200614:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176030.200620:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176030.200625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176030.200629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176030.200632:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176030.200635:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176030.200638:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176030.776734:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176030.776742:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176030.776748:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.0:1786176031.096630:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176031.096639:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176031.096645:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:2.0:1786176031.224632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176031.224640:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176031.224642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176031.224648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176031.224651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176031.224655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176031.224658:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176031.224660:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176031.224663:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176031.800726:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176031.800732:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176031.800736:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176032.248656:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176032.248666:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176032.248670:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176032.248678:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176032.248683:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176032.248688:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176032.248692:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176032.248696:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176032.248700:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176032.824770:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176032.824783:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176032.824789:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176033.272604:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176033.272612:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176033.272615:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176033.272622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176033.272627:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176033.272631:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176033.272635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176033.272638:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176033.272654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176033.848693:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176033.848697:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176033.848701:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176034.296642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176034.296649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176034.296651:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176034.296655:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176034.296659:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176034.296662:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176034.296668:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176034.296672:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176034.296676:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176034.297190:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176034.297196:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786176034.297771:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176034.297773:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786176034.297954:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176034.297956:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786176034.298154:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176034.298156:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786176034.298335:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176034.298338:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000020:00000001:2.0:1786176034.872670:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176034.872674:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176034.872677:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176035.320601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176035.320609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176035.320612:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176035.320619:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176035.320624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176035.320628:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176035.320631:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176035.320634:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176035.320637:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176035.896795:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176035.896801:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176035.896805:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176036.344597:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176036.344603:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176036.344605:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176036.344609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176036.344611:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176036.344614:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176036.344616:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176036.344618:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176036.344620:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176036.920846:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176036.920852:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176036.920857:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176037.368658:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176037.368667:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176037.368669:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176037.368676:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176037.368680:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176037.368685:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176037.368689:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176037.368693:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176037.368696:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176037.944795:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176037.944801:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176037.944806:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176038.392630:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176038.392638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176038.392641:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176038.392648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176038.392652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176038.392656:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176038.392659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176038.392663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176038.392666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176038.968724:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176038.968729:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176038.968733:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176039.416670:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176039.416685:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176039.416691:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176039.416708:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176039.416718:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176039.416728:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176039.416747:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176039.416756:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176039.416766:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176039.417634:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176039.417644:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786176039.418774:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176039.418776:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786176039.419122:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176039.419126:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786176039.419437:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176039.419439:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786176039.419728:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176039.419730:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000020:00000001:2.0:1786176039.992759:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176039.992766:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176039.992772:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176040.440675:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176040.440688:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176040.440692:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176040.440704:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176040.440711:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176040.440718:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176040.440723:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176040.440730:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176040.440736:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176041.016746:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176041.016754:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176041.016759:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176041.464628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176041.464645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176041.464646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176041.464651:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176041.464654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176041.464658:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176041.464660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176041.464663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176041.464665:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176042.040677:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176042.040681:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176042.040684:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176042.488587:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176042.488594:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176042.488596:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176042.488601:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176042.488605:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176042.488619:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176042.488622:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176042.488624:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176042.488627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176043.064656:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176043.064667:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176043.064672:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176043.512702:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176043.512712:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176043.512715:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176043.512722:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176043.512727:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176043.512732:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176043.512736:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176043.512740:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176043.512744:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176044.088664:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176044.088668:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176044.088670:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176044.536632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176044.536641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176044.536645:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176044.536653:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176044.536659:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176044.536664:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176044.536670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176044.536675:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176044.536680:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176044.537659:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176044.537670:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1786176044.539054:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176044.539058:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1786176044.539427:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176044.539431:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1786176044.539814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176044.539817:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1786176044.540335:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176044.540346:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000020:00000001:2.0:1786176045.112823:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176045.112831:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176045.112837:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176045.560601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176045.560607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176045.560609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176045.560614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176045.560617:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176045.560620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176045.560623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176045.560625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176045.560627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176046.136679:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176046.136684:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176046.136688:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176046.584640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176046.584649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176046.584652:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176046.584660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176046.584664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176046.584669:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176046.584673:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176046.584676:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176046.584679:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176047.160923:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176047.160930:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176047.160936:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176047.608572:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176047.608579:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176047.608581:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176047.608585:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176047.608588:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176047.608590:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176047.608593:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176047.608595:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176047.608596:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176048.184671:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176048.184674:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176048.184677:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176048.632580:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176048.632586:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176048.632588:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176048.632592:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176048.632594:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176048.632597:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176048.632599:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176048.632601:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176048.632603:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176049.208700:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176049.208703:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176049.208706:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176049.656665:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176049.656678:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176049.656687:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176049.656699:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176049.656709:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176049.656720:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176049.656730:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176049.656739:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176049.656751:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176049.657491:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176049.657503:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176049.658775:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176049.658784:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176049.659459:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176049.659461:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176049.659984:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176049.659987:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176049.660333:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176049.660336:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:2.0:1786176050.232782:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176050.232788:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176050.232810:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176050.680617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176050.680628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176050.680632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176050.680641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176050.680647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176050.680652:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176050.680657:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176050.680661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176050.680666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176051.256753:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176051.256759:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176051.256764:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176051.704596:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176051.704604:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176051.704606:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176051.704612:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176051.704616:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176051.704619:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176051.704623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176051.704625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176051.704629:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176052.280722:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176052.280727:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176052.280731:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176052.728609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176052.728617:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176052.728620:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176052.728627:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176052.728631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176052.728635:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176052.728639:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176052.728642:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176052.728645:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176053.304773:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176053.304782:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176053.304789:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176053.752645:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176053.752657:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176053.752662:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176053.752671:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176053.752677:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176053.752684:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176053.752689:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176053.752694:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176053.752699:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176054.328728:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176054.328733:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176054.328738:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176054.776650:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176054.776661:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176054.776671:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176054.776682:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176054.776694:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176054.776702:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176054.776709:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176054.776716:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176054.776722:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176054.777763:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176054.777770:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176054.779191:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176054.779197:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176054.779707:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176054.779714:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176054.780340:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176054.780345:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176054.781011:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176054.781014:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:2.0:1786176055.352809:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176055.352813:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176055.352815:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176055.800641:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176055.800652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176055.800656:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176055.800663:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176055.800669:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176055.800673:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176055.800678:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176055.800681:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176055.800686:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176056.376930:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176056.376940:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176056.376946:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176056.824644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176056.824653:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176056.824656:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176056.824663:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176056.824668:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176056.824672:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176056.824676:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176056.824679:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176056.824683:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176057.400891:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176057.400899:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176057.400905:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176057.848612:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176057.848620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176057.848622:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176057.848629:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176057.848633:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176057.848637:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176057.848640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176057.848643:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176057.848646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176058.424955:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176058.424964:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176058.424970:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176058.872666:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176058.872677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176058.872680:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176058.872688:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176058.872692:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176058.872697:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176058.872701:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176058.872704:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176058.872707:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176059.448885:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176059.448906:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176059.448912:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176059.896597:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176059.896605:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176059.896608:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176059.896616:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176059.896623:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176059.896645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176059.896653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176059.896658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176059.896664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176059.897579:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176059.897583:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176059.898830:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176059.898833:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176059.899083:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176059.899085:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176059.899331:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176059.899333:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176059.899661:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176059.899663:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:2.0:1786176060.472806:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176060.472809:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176060.472812:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176060.920629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176060.920637:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176060.920640:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176060.920648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176060.920652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176060.920656:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176060.920659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176060.920663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176060.920666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176061.496667:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176061.496670:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176061.496672:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176061.944635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176061.944643:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176061.944645:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176061.944650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176061.944653:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176061.944655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176061.944658:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176061.944660:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176061.944661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176062.520808:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176062.520814:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176062.520819:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176062.968686:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176062.968695:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176062.968697:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176062.968703:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176062.968707:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176062.968710:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176062.968713:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176062.968716:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176062.968718:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176063.544797:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176063.544801:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176063.544804:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176063.992609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176063.992615:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176063.992618:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176063.992622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176063.992625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176063.992629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176063.992631:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176063.992634:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176063.992636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176064.568823:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176064.568830:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176064.568835:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176065.016660:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176065.016671:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176065.016680:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176065.016692:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176065.016731:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176065.016740:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176065.016767:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176065.016775:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176065.016786:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176065.017640:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176065.017650:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176065.018697:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176065.018700:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176065.019105:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176065.019108:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176065.019447:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176065.019451:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176065.019930:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176065.019934:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:2.0:1786176065.592787:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176065.592793:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176065.592797:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176066.040761:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176066.040773:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176066.040776:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176066.040784:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176066.040790:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176066.040795:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176066.040800:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176066.040804:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176066.040808:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176066.616804:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176066.616823:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176066.616827:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176067.064667:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176067.064703:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176067.064726:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176067.064736:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176067.064743:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176067.064750:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176067.064754:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176067.064757:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176067.064762:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176067.640872:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176067.640879:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176067.640884:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176068.088646:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176068.088655:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176068.088658:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176068.088664:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176068.088669:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176068.088696:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176068.088700:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176068.088704:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176068.088708:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176068.664808:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176068.664813:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176068.664818:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176069.112655:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176069.112695:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176069.112724:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176069.112732:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176069.112736:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176069.112742:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176069.112747:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176069.112752:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176069.112758:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176069.688768:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176069.688776:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176069.688783:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176070.136605:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176070.136613:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176070.136616:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176070.136623:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176070.136627:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176070.136649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176070.136681:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176070.136690:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176070.136714:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176070.137624:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176070.138222:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176070.138926:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176070.139271:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176070.139591:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786176070.712782:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176070.712786:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176070.712791:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176071.160819:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176071.160831:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176071.160833:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176071.160839:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176071.160843:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176071.160846:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176071.160849:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176071.160852:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176071.160855:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176071.736766:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176071.736771:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176071.736775:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176072.184727:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176072.184735:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176072.184737:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176072.184743:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176072.184747:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176072.184751:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176072.184754:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176072.184758:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176072.184761:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176072.760792:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176072.760799:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176072.760803:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176073.208733:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176073.208739:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176073.208741:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176073.208745:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176073.208748:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176073.208751:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176073.208753:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176073.208755:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176073.208757:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176073.784639:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176073.784643:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176073.784646:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176074.232782:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176074.232794:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176074.232798:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176074.232808:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176074.232815:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176074.232821:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176074.232826:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176074.232831:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176074.232836:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176074.808844:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176074.808850:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176074.808855:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176075.256756:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176075.256765:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176075.256770:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176075.256780:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176075.256786:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176075.256795:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176075.256799:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176075.256804:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176075.256815:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176075.258017:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176075.258025:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176075.259499:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176075.259552:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176075.260129:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176075.260133:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176075.260663:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176075.260667:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176075.261084:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176075.261087:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:2.0:1786176075.832686:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176075.832691:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176075.832695:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176076.280741:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176076.280750:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176076.280752:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176076.280757:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176076.280767:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176076.280774:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176076.280777:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176076.280779:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176076.280781:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176076.856845:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176076.856852:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176076.856857:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176077.304624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176077.304631:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176077.304633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176077.304638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176077.304641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176077.304644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176077.304646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176077.304648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176077.304651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176077.880719:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176077.880725:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176077.880730:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176078.328681:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176078.328689:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176078.328691:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176078.328696:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176078.328700:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176078.328704:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176078.328707:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176078.328710:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176078.328713:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176078.904771:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176078.904776:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176078.904778:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176079.352629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176079.352635:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176079.352637:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176079.352641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176079.352644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176079.352647:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176079.352650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176079.352652:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176079.352654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176079.928744:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176079.928753:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176079.928759:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176080.377734:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176080.377744:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786176080.379005:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176080.379012:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786176080.379376:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176080.379379:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786176080.379765:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176080.379768:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786176080.380026:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176080.380028:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:2.0:1786176080.952719:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176080.952726:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176080.952730:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176081.976656:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176081.976660:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176081.976663:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176083.000775:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176083.000784:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176083.000791:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176084.024691:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176084.024695:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176084.024698:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176085.048748:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176085.048757:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176085.048764:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176085.497211:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176085.497217:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786176085.498025:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176085.498027:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786176085.498239:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176085.498241:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786176085.498453:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176085.498456:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786176085.498829:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176085.498832:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:2.0:1786176086.072699:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176086.072705:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176086.072710:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176087.096794:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176087.096801:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176087.096807:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176088.120756:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176088.120765:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176088.120771:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176089.144706:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176089.144711:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176089.144715:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176090.168726:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176090.168733:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176090.168739:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176090.617381:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176090.617388:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786176090.618742:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176090.618745:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786176090.619098:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176090.619101:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786176090.619302:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176090.619303:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786176090.619693:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176090.619695:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:2.0:1786176091.192630:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176091.192633:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176091.192636:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176092.216648:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176092.216651:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176092.216654:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176093.240756:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176093.240781:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176093.240793:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176094.264822:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176094.264828:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176094.264833:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176095.288745:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176095.288750:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176095.288753:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176095.737161:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176095.737165:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176095.737963:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176095.737966:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176095.738286:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176095.738289:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176095.738641:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176095.738644:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176095.738959:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176095.738962:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:2.0:1786176096.312673:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176096.312678:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176096.312682:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176097.336902:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176097.336909:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176097.336914:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176098.360679:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176098.360683:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176098.360686:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176099.384727:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176099.384731:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176099.384734:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176100.408733:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176100.408736:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176100.408739:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176100.857199:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176100.857204:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176100.857970:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176100.857975:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176100.858242:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176100.858243:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176100.858498:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176100.858500:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176100.858875:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176100.858877:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:2.0:1786176101.432688:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176101.432691:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176101.432694:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176102.456698:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176102.456703:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176102.456706:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176103.480676:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176103.480680:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176103.480683:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176104.504655:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176104.504660:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176104.504662:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176105.528817:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176105.528822:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176105.528825:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176105.977476:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176105.977482:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176105.978556:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176105.978560:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176105.978916:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176105.978919:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176105.979195:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176105.979198:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176105.979666:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176105.979670:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:2.0:1786176106.552676:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176106.552679:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176106.552682:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176107.576671:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176107.576674:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176107.576677:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176108.600730:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176108.600737:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176108.600742:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176109.624733:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176109.624739:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176109.624744:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176110.648702:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176110.648708:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176110.648713:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176111.097103:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176111.097108:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786176111.098067:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176111.098070:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786176111.098391:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176111.098394:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786176111.098710:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176111.098713:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786176111.099008:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176111.099011:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:2.0:1786176111.672659:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176111.672663:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176111.672666:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176112.696661:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176112.696664:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176112.696668:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176113.720727:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176113.720732:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176113.720735:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176114.744814:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176114.744820:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176114.744824:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176115.768713:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176115.768717:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176115.768719:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176116.217121:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176116.217125:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176116.217725:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176116.217728:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176116.217941:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176116.217943:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176116.218135:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176116.218137:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176116.218397:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176116.218399:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:2.0:1786176116.792750:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176116.792754:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176116.792757:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176117.816735:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176117.816740:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176117.816743:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176118.840835:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176118.840841:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176118.840846:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176119.864690:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176119.864723:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176119.864726:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176120.888701:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176120.888705:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176120.888708:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176121.337252:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176121.337262:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176121.338094:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176121.338096:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176121.338439:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176121.338441:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176121.338683:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176121.338685:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176121.338974:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176121.338976:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:2.0:1786176121.912733:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176121.912737:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176121.912739:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176122.936763:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176122.936770:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176122.936774:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176123.960690:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176123.960693:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176123.960696:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176124.984708:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176124.984713:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176124.984716:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176126.008710:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176126.008715:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176126.008719:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176126.457397:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176126.457405:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176126.458348:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176126.458350:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176126.458746:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176126.458748:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176126.458975:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176126.458977:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176126.459354:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176126.459356:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:2.0:1786176127.032710:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176127.032714:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176127.032717:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176128.056651:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176128.056655:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176128.056657:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176129.080815:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176129.080821:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176129.080824:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176130.104644:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176130.104649:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176130.104651:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176131.128664:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176131.128667:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176131.128669:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176131.577192:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176131.577198:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786176131.577844:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176131.577846:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786176131.578046:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176131.578048:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786176131.578281:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176131.578283:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1786176131.578471:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176131.578472:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000020:00000001:2.0:1786176132.152769:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176132.152773:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176132.152777:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176133.176699:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176133.176702:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176133.176705:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176134.200909:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176134.200916:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176134.200921:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176135.224779:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176135.224784:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176135.224788:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176136.248747:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176136.248751:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176136.248754:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176136.697176:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176136.697533:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176136.697759:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176136.697934:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176136.698102:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786176137.272735:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176137.272738:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176137.272741:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176138.296747:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176138.296752:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176138.296755:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176139.320774:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176139.320780:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176139.320785:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176140.344836:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176140.344842:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176140.344849:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176141.368716:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176141.368722:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176141.368727:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176141.817935:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176141.817941:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176141.819309:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176141.819323:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176141.820620:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176141.820624:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176141.821120:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176141.821125:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176141.821579:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176141.821582:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:2.0:1786176142.392807:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176142.392814:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176142.392819:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176143.416726:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176143.416732:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176143.416737:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176144.440815:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176144.440821:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176144.440826:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176145.464760:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176145.464769:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176145.464775:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.0:1786176145.784647:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176145.784658:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176145.784665:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000020:00000001:2.0:1786176146.488840:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176146.488847:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176146.488852:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176146.937763:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176146.937773:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176146.938636:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176146.938638:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176146.939032:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176146.939034:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176146.939404:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176146.939408:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176146.939777:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176146.939781:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:2.0:1786176147.512828:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176147.512831:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176147.512834:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176148.536779:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176148.536785:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176148.536790:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176149.560744:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176149.560749:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176149.560755:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176150.584832:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176150.584839:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176150.584844:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176151.608913:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176151.608920:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176151.608925:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176152.056734:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176152.056743:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176152.056745:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176152.056748:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176152.056752:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176152.056760:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6375
00000400:00000200:2.0:1786176152.056768:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176152.056774:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176152.056778:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176152.056781:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.056796:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176152.056799:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.056801:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176152.056805:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176152.056807:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176152.056810:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176152.056814:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176152.056816:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176152.056818:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176152.056821:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.056822:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176152.056824:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.056826:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176152.056829:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176152.056832:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176152.056834:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176152.056837:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176152.056840:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176152.056843:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.056845:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176152.056847:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176152.056858:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786176152.056882:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176152.056889:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176152.056890:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.056894:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176152.056897:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176152.056902:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176152.056911:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786176152.056918:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176152.056930:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856218112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176152.056947:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176152.056951:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176152.056956:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856218112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176152.056979:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176152.056982:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176152.056985:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176152.056987:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176152.056989:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176152.056992:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176152.056994:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6375
00000400:00000200:2.0:1786176152.057000:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176152.057003:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176152.057005:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176152.057009:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057012:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176152.057015:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057017:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176152.057020:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176152.057022:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176152.057024:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176152.057026:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176152.057031:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176152.057034:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176152.057040:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057045:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176152.057048:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057051:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176152.057056:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176152.057065:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057073:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176152.057079:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176152.057089:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176152.057098:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.057107:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176152.057114:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176152.057123:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.057131:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176152.057141:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786176152.057150:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176152.057159:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176152.057163:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.057170:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176152.057174:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176152.057188:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176152.057198:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786176152.057206:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176152.057219:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856218240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176152.057242:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176152.057246:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176152.057253:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856218240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176152.057270:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176152.057275:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176152.057282:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176152.057286:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176152.057289:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176152.057295:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176152.057304:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6375
00000400:00000200:2.0:1786176152.057317:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176152.057326:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176152.057333:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176152.057342:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057348:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176152.057354:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057359:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176152.057366:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176152.057367:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176152.057369:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176152.057375:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176152.057382:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176152.057385:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176152.057394:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:0.1:1786176152.057397:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176152.057398:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1786176152.057405:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:2.0:1786176152.057406:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057414:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176152.057421:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176152.057430:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057435:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176152.057443:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176152.057449:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176152.057457:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.057460:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176152.057463:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176152.057468:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.057472:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176152.057478:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786176152.057484:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176152.057492:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176152.057496:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.057500:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176152.057502:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176152.057508:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176152.057549:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786176152.057557:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176152.057567:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856218368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176152.057586:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176152.057590:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176152.057597:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856218368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176152.057613:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176152.057622:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176152.057630:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176152.057635:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176152.057638:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176152.057647:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176152.057653:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6375
00000400:00000200:2.0:1786176152.057659:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176152.057666:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176152.057673:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176152.057677:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057678:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176152.057680:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057682:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176152.057684:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176152.057687:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176152.057691:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176152.057695:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176152.057699:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176152.057706:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176152.057712:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057718:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176152.057724:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057727:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176152.057729:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176152.057734:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057739:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176152.057744:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176152.057749:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176152.057755:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.057759:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176152.057763:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176152.057771:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.057775:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176152.057779:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176152.057785:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176152.057790:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176152.057794:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.057799:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176152.057801:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176152.057806:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176152.057812:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786176152.057818:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176152.057830:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856218496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176152.057844:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176152.057850:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176152.057878:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856218496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176152.057900:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176152.057906:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176152.057915:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176152.057921:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176152.057926:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176152.057933:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176152.057941:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6375
00000400:00000200:2.0:1786176152.057952:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176152.057960:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176152.057964:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176152.057975:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057984:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176152.057992:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.057995:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176152.058002:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176152.058005:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176152.058008:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176152.058013:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176152.058020:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176152.058026:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176152.058035:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.058040:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176152.058045:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176152.058049:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176152.058054:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786176152.058060:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176152.058065:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176152.058069:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176152.058075:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176152.058084:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.058090:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176152.058096:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176152.058105:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.058114:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176152.058125:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786176152.058135:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176152.058145:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176152.058151:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176152.058156:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176152.058159:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176152.058164:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176152.058171:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786176152.058179:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176152.058192:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856218624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176152.058210:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176152.058217:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176152.058227:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856218624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176152.058261:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176152.058268:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6385)
00000800:00000200:0.1:1786176152.058330:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176152.058332:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176152.058616:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176152.058618:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176152.058901:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176152.058903:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176152.059260:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176152.059262:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:2.0:1786176152.632758:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176152.632766:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176152.632772:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176153.656744:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176153.656749:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176153.656753:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176154.680697:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176154.680702:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176154.680706:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176155.704816:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176155.704821:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176155.704846:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176156.728896:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176156.728904:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176156.728910:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176157.176756:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176157.176770:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176157.176776:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176157.176781:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176157.176788:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176157.176797:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6380
00000400:00000200:2.0:1786176157.176809:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176157.176873:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176157.176877:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176157.176881:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.176885:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176157.176888:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.176890:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176157.176894:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176157.176897:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176157.176900:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176157.176904:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176157.176906:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176157.176908:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176157.176910:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.176912:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176157.176913:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.176915:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176157.176918:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176157.176921:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176157.176923:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176157.176925:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176157.176930:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176157.176933:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.176937:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176157.176939:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176157.176950:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000020:00000040:2.0:1786176157.176954:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176157.176959:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176157.176961:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.176965:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176157.176969:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176157.176974:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176157.176982:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786176157.176991:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176157.177001:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856218752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176157.177014:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176157.177019:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176157.177025:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856218752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176157.177053:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176157.177058:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176157.177062:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176157.177064:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176157.177066:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176157.177069:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176157.177072:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6380
00000400:00000200:2.0:1786176157.177078:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176157.177082:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176157.177086:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176157.177090:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177094:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176157.177097:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177100:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176157.177106:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176157.177108:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176157.177112:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176157.177114:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176157.177119:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176157.177121:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176157.177128:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177133:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176157.177137:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177142:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176157.177147:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176157.177151:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177156:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176157.177158:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176157.177164:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176157.177172:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.177176:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176157.177183:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176157.177188:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.177193:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176157.177201:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000020:00000040:2.0:1786176157.177205:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176157.177208:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176157.177212:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.177218:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176157.177219:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176157.177222:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176157.177230:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786176157.177235:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176157.177245:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856218880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176157.177273:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176157.177278:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176157.177285:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856218880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176157.177304:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176157.177309:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176157.177313:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176157.177317:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176157.177321:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176157.177326:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176157.177332:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6380
00000400:00000200:2.0:1786176157.177342:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176157.177349:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176157.177354:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176157.177362:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177367:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176157.177375:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177380:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176157.177383:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176157.177387:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176157.177391:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176157.177393:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176157.177399:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176157.177401:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176157.177410:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177414:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176157.177416:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177420:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176157.177426:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176157.177431:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177456:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176157.177461:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176157.177465:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176157.177471:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.177477:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176157.177481:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176157.177486:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.177492:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176157.177498:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000020:00000040:2.0:1786176157.177506:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176157.177537:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176157.177539:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.177541:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176157.177542:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176157.177545:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176157.177551:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786176157.177558:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176157.177573:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:2.0:1786176157.177574:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856219008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1786176157.177578:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1786176157.177603:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176157.177619:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176157.177633:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856219008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176157.177654:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176157.177661:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176157.177666:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176157.177671:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176157.177674:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176157.177696:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176157.177702:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6380
00000400:00000200:2.0:1786176157.177713:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176157.177721:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176157.177726:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176157.177733:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177736:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176157.177741:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177745:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176157.177748:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176157.177750:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176157.177756:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176157.177760:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176157.177764:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176157.177769:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176157.177775:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177780:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176157.177785:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177789:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176157.177793:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176157.177802:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176157.177825:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176157.177833:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176157.177840:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176157.177847:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.177861:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176157.177866:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176157.177875:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.177881:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176157.177892:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000020:00000040:2.0:1786176157.177900:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176157.177910:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176157.177915:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.177923:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176157.177949:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176157.177956:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176157.177967:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:2.0:1786176157.177978:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176157.178010:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856219136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176157.178021:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176157.178026:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176157.178033:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856219136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176157.178047:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176157.178051:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176157.178057:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176157.178062:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176157.178065:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176157.178078:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176157.178084:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6380
00000400:00000200:2.0:1786176157.178096:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176157.178100:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176157.178103:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176157.178109:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.178114:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176157.178117:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.178123:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176157.178126:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176157.178129:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176157.178132:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176157.178135:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176157.178140:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176157.178146:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176157.178153:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.178158:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176157.178166:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176157.178169:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176157.178175:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786176157.178182:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176157.178185:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176157.178189:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176157.178194:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176157.178198:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.178200:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176157.178204:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176157.178208:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.178213:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176157.178220:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786176157.178226:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176157.178229:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176157.178233:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176157.178235:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176157.178239:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176157.178241:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176157.178248:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786176157.178254:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176157.178262:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856219264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176157.178269:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176157.178271:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176157.178274:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856219264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176157.178279:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176157.178283:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6390)
00000800:00000200:3.1:1786176157.178447:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176157.178449:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786176157.178695:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176157.178697:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786176157.179016:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176157.179020:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786176157.179373:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176157.179376:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:2.0:1786176157.752867:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176157.752878:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176157.752886:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176158.776675:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176158.776679:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176158.776681:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176159.800993:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176159.801003:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176159.801008:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176160.824720:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176160.824726:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176160.824729:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176161.848662:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176161.848666:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176161.848670:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176162.296627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176162.296635:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176162.296638:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176162.296646:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176162.296653:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176162.296658:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176162.296663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176162.296668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176162.296673:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176162.297215:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176162.297221:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176162.297812:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176162.297814:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176162.298006:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176162.298007:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176162.298179:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176162.298181:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176162.298352:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176162.298354:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:2.0:1786176162.872672:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176162.872675:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176162.872678:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176163.320645:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176163.320653:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176163.320656:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176163.320662:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176163.320666:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176163.320671:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176163.320674:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176163.320677:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176163.320681:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176163.896828:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176163.896834:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176163.896839:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176164.344648:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176164.344658:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176164.344660:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176164.344667:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176164.344672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176164.344676:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176164.344679:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176164.344683:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176164.344686:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176164.920648:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176164.920652:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176164.920654:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176165.368568:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176165.368574:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176165.368576:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176165.368581:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176165.368583:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176165.368586:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176165.368589:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176165.368591:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176165.368593:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176165.944822:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176165.944825:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176165.944828:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176166.392634:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176166.392644:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176166.392646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176166.392652:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176166.392656:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176166.392661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176166.392665:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176166.392668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176166.392671:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176166.968698:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176166.968701:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176166.968704:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176167.416579:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176167.416586:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176167.416588:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176167.416593:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176167.416595:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176167.416599:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176167.416601:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176167.416604:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176167.416606:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176167.417184:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176167.417189:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176167.417974:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176167.417975:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176167.418161:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176167.418162:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176167.418334:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176167.418337:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176167.418660:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176167.418662:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000020:00000001:2.0:1786176167.992826:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176167.992832:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176167.992837:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176168.440586:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176168.440593:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176168.440594:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176168.440598:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176168.440601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176168.440603:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176168.440606:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176168.440608:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176168.440610:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176169.016793:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176169.016797:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176169.016800:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176169.464673:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176169.464681:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176169.464683:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176169.464689:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176169.464746:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176169.464750:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176169.464754:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176169.464757:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176169.464760:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176170.040808:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176170.040816:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176170.040822:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176170.488652:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176170.488660:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176170.488664:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176170.488671:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176170.488675:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176170.488679:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176170.488714:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176170.488718:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176170.488722:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176171.064653:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176171.064658:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176171.064662:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176171.512610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176171.512616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176171.512618:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176171.512622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176171.512625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176171.512628:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176171.512630:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176171.512632:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176171.512634:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176172.088788:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176172.088792:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176172.088795:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176172.536638:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176172.536652:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176172.536659:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176172.536685:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176172.536695:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176172.536707:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6395
00000400:00000200:2.0:1786176172.536722:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176172.536731:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176172.536735:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176172.536748:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.536752:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176172.536755:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.536759:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176172.536764:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176172.536767:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176172.536770:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176172.536774:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176172.536777:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176172.536779:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176172.536782:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.536784:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176172.536785:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.536787:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176172.536790:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176172.536794:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176172.536796:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176172.536800:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176172.536804:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176172.536807:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.536810:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176172.536812:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176172.536824:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176172.536828:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176172.536833:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176172.536835:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.536839:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176172.536843:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176172.536849:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176172.536859:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786176172.536868:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176172.536879:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856220672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176172.536891:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176172.536896:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176172.536901:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856220672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176172.536926:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176172.536930:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176172.536933:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176172.536935:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176172.536937:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176172.536940:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176172.536942:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6395
00000400:00000200:2.0:1786176172.536946:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176172.536949:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176172.536951:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176172.536954:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.536956:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176172.536958:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.536959:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176172.536961:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176172.536962:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176172.536964:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176172.536977:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176172.536979:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176172.536981:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176172.536984:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.536986:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176172.536988:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.536989:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176172.536991:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176172.536994:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176172.536996:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176172.536999:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176172.537005:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176172.537008:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.537010:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176172.537016:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176172.537020:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.537023:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176172.537026:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786176172.537030:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176172.537032:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176172.537034:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.537036:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176172.537037:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176172.537039:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176172.537043:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786176172.537047:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176172.537059:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856220800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176172.537074:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176172.537079:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176172.537088:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856220800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176172.537154:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176172.537159:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176172.537172:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176172.537180:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176172.537184:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176172.537193:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176172.537209:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6395
00000400:00000200:2.0:1786176172.537228:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176172.537235:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176172.537239:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176172.537252:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.537256:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176172.537258:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.537260:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176172.537263:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176172.537268:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176172.537274:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176172.537278:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176172.537288:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176172.537299:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176172.537308:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.537312:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176172.537323:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.537327:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176172.537334:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176172.537345:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176172.537350:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176172.537354:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176172.537363:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176172.537372:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.537382:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176172.537392:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176172.537406:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.537419:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176172.537426:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786176172.537440:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176172.537444:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176172.537445:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.537455:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176172.537461:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176172.537464:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176172.537468:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786176172.537477:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176172.537495:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856220928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176172.537543:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176172.537545:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176172.537551:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856220928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176172.537560:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176172.537564:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176172.537568:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176172.537570:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176172.537572:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176172.537576:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176172.537580:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6395
00000400:00000200:2.0:1786176172.537584:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176172.537589:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176172.537591:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176172.537603:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.537605:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176172.537612:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.537623:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176172.537633:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176172.537636:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176172.537643:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176172.537653:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176172.537657:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176172.537682:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176172.537690:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.537700:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176172.537712:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.537716:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:0.1:1786176172.537721:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1786176172.537727:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000800:00000200:0.1:1786176172.537730:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000100:00080000:2.0:1786176172.537739:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176172.537751:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176172.537755:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176172.537766:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176172.537771:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.537776:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176172.537781:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176172.537788:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.537797:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176172.537815:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786176172.537821:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176172.537830:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176172.537834:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.537846:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176172.537852:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176172.537857:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176172.537870:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786176172.537879:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176172.537894:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856221056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176172.537928:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176172.537933:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176172.537943:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856221056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176172.537973:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176172.537979:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176172.537988:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176172.538001:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176172.538005:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176172.538010:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176172.538020:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6395
00000400:00000200:2.0:1786176172.538031:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176172.538038:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176172.538050:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176172.538066:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.538072:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176172.538081:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.538091:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176172.538096:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176172.538100:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176172.538107:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176172.538118:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176172.538135:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176172.538141:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176172.538153:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.538163:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176172.538173:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176172.538181:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176172.538192:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786176172.538205:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176172.538212:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176172.538214:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176172.538218:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176172.538224:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.538226:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176172.538229:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176172.538235:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.538240:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176172.538257:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786176172.538265:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176172.538268:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176172.538271:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176172.538279:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176172.538284:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176172.538289:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176172.538299:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786176172.538305:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176172.538317:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856221184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176172.538369:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176172.538373:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176172.538380:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856221184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176172.538407:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176172.538418:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6405)
00000800:00000200:0.1:1786176172.538657:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176172.538659:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786176172.538944:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176172.538962:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786176172.539230:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176172.539232:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786176172.539417:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176172.539419:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000020:00000001:2.0:1786176173.112912:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176173.112919:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176173.112924:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176174.136781:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176174.136786:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176174.136789:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176175.160757:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176175.160762:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176175.160765:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176176.184793:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176176.184801:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176176.184807:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176177.208824:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176177.208829:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176177.208832:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176177.656657:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176177.656672:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176177.656704:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176177.656711:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176177.656716:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176177.656726:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6400
00000400:00000200:2.0:1786176177.656740:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176177.656749:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176177.656753:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176177.656757:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.656761:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176177.656763:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.656766:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176177.656770:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176177.656772:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176177.656775:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176177.656778:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176177.656780:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176177.656783:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176177.656786:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.656787:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176177.656789:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.656791:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176177.656793:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176177.656796:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176177.656798:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176177.656801:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176177.656804:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176177.656807:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.656809:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176177.656811:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176177.656821:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786176177.656824:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176177.656829:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176177.656831:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.656834:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176177.656838:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176177.656844:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176177.656853:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000001:2.0:1786176177.656860:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176177.656871:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856221312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176177.656881:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176177.656897:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176177.656901:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856221312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176177.656923:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176177.656926:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176177.656929:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176177.656930:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176177.656932:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176177.656934:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176177.656936:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6400
00000400:00000200:2.0:1786176177.656939:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176177.656941:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176177.656943:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176177.656946:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.656948:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176177.656949:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.656951:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176177.656953:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176177.656955:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176177.656956:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176177.656958:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176177.656961:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176177.656964:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176177.656966:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.656968:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176177.656970:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.656972:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176177.656990:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176177.656993:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176177.656995:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176177.656996:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176177.657003:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176177.657008:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.657011:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176177.657018:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176177.657023:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.657025:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176177.657030:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000020:00000040:2.0:1786176177.657034:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176177.657049:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176177.657052:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.657058:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176177.657060:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176177.657064:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176177.657072:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786176177.657077:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176177.657090:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856221440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176177.657112:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176177.657117:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176177.657124:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856221440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176177.657147:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176177.657151:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176177.657155:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176177.657157:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176177.657161:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176177.657165:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176177.657170:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6400
00000400:00000200:2.0:1786176177.657175:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176177.657179:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176177.657183:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176177.657190:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657193:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176177.657197:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657204:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176177.657207:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176177.657209:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176177.657215:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176177.657235:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176177.657239:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176177.657245:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176177.657253:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657261:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176177.657267:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657273:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176177.657279:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176177.657286:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657293:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176177.657297:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176177.657301:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176177.657307:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.657311:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176177.657313:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176177.657316:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.657320:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176177.657326:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000020:00000040:2.0:1786176177.657331:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176177.657334:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176177.657335:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.657336:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176177.657338:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176177.657341:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176177.657347:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786176177.657352:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176177.657357:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856221568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176177.657369:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176177.657371:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176177.657377:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856221568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176177.657387:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176177.657390:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176177.657395:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176177.657399:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176177.657403:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176177.657407:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176177.657413:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6400
00000400:00000200:2.0:1786176177.657420:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176177.657426:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176177.657429:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176177.657434:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657445:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176177.657447:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657448:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176177.657450:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176177.657451:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176177.657452:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176177.657454:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176177.657458:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176177.657461:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176177.657470:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657473:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176177.657478:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657484:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176177.657486:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176177.657489:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657492:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176177.657493:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176177.657498:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176177.657502:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.657506:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176177.657509:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176177.657530:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.657531:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176177.657535:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786176177.657537:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176177.657539:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176177.657540:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.657595:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176177.657599:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176177.657602:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176177.657610:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786176177.657632:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176177.657639:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856221696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176177.657678:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176177.657682:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176177.657690:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856221696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176177.657696:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176177.657699:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176177.657708:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176177.657710:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176177.657713:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:0.1:1786176177.657715:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176177.657716:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176177.657723:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6400
00000800:00000200:0.1:1786176177.657725:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000400:00000200:2.0:1786176177.657728:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176177.657735:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176177.657737:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176177.657743:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657748:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176177.657750:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657756:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176177.657759:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176177.657761:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176177.657762:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176177.657765:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176177.657771:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176177.657774:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176177.657781:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657787:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176177.657790:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657793:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176177.657796:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786176177.657802:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176177.657804:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176177.657810:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176177.657813:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176177.657815:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.657820:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176177.657822:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176177.657826:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.657828:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176177.657834:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786176177.657839:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176177.657844:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176177.657847:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176177.657851:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176177.657854:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176177.657856:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176177.657861:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786176177.657866:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176177.657873:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856221824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176177.657902:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176177.657905:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176177.657912:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856221824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176177.657920:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176177.657925:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6411)
00000800:00000200:0.1:1786176177.658487:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176177.658490:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176177.659049:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176177.659052:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176177.659434:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176177.659438:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176177.659746:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176177.659748:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:2.0:1786176178.232758:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176178.232764:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176178.232769:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176179.256806:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176179.256810:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176179.256814:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176180.280715:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176180.280720:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176180.280724:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176181.304757:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176181.304763:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176181.304767:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176182.328742:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176182.328746:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176182.328748:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176182.776595:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176182.776605:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176182.776611:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176182.776617:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176182.776624:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176182.776634:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6406
00000400:00000200:2.0:1786176182.776644:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176182.776651:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176182.776654:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176182.776658:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.776662:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176182.776664:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.776666:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176182.776669:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176182.776670:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176182.776672:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176182.776675:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176182.776676:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176182.776678:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176182.776680:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.776682:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176182.776683:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.776685:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176182.776687:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176182.776690:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176182.776691:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176182.776693:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176182.776696:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176182.776698:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.776699:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176182.776701:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176182.776708:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786176182.776711:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176182.776715:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176182.776716:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.776718:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176182.776721:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176182.776724:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176182.776730:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786176182.776735:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176182.776742:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856221952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176182.776751:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176182.776754:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176182.776758:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856221952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176182.776777:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176182.776780:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176182.776783:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176182.776784:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176182.776785:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176182.776787:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176182.776790:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6406
00000400:00000200:2.0:1786176182.776793:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176182.776795:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176182.776796:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176182.776799:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.776800:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176182.776803:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.776804:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176182.776806:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176182.776807:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176182.776808:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176182.776810:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176182.776812:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176182.776815:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176182.776819:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.776823:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176182.776826:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.776828:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176182.776831:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176182.776837:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176182.776842:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176182.776847:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176182.776852:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176182.776858:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.776861:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176182.776865:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176182.776870:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.776874:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176182.776877:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786176182.776881:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176182.776884:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176182.776887:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.776889:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176182.776890:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176182.776891:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176182.776896:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786176182.776899:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176182.776906:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856222080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176182.776915:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176182.776919:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176182.776922:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856222080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176182.776938:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176182.776941:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176182.776947:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176182.776950:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176182.776952:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176182.776958:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176182.776964:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6406
00000400:00000200:2.0:1786176182.776972:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176182.776977:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176182.776979:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176182.776984:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.776987:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176182.776989:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.776990:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176182.776991:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176182.776994:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176182.776998:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176182.777000:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176182.777002:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176182.777006:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176182.777011:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777015:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176182.777019:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777022:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176182.777026:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176182.777032:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777036:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176182.777038:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176182.777048:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176182.777053:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.777055:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176182.777059:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176182.777062:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.777064:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176182.777070:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786176182.777072:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176182.777075:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176182.777076:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.777077:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176182.777079:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176182.777081:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176182.777093:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000800:00000200:0.1:1786176182.777094:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176182.777098:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176182.777099:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00100000:2.0:1786176182.777107:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856222208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176182.777118:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176182.777122:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176182.777129:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856222208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176182.777140:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176182.777145:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176182.777149:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176182.777152:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176182.777154:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176182.777159:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176182.777162:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6406
00000400:00000200:2.0:1786176182.777165:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176182.777167:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176182.777168:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176182.777171:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777175:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176182.777177:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777181:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176182.777184:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176182.777186:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176182.777189:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176182.777192:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176182.777197:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176182.777199:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176182.777205:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777207:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176182.777209:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777215:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176182.777219:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176182.777225:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777228:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176182.777230:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176182.777233:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176182.777238:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.777242:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176182.777246:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176182.777253:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.777257:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176182.777263:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176182.777269:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176182.777275:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176182.777278:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.777283:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176182.777286:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176182.777290:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176182.777296:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786176182.777303:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176182.777312:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856222336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176182.777331:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176182.777335:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176182.777341:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856222336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176182.777354:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176182.777357:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176182.777363:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176182.777366:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176182.777369:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176182.777374:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176182.777378:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6406
00000400:00000200:2.0:1786176182.777386:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176182.777390:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176182.777394:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176182.777408:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777412:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176182.777417:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777420:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176182.777424:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176182.777428:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176182.777430:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176182.777433:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176182.777436:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176182.777438:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176182.777443:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777445:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176182.777450:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777453:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176182.777458:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786176182.777462:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176182.777467:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176182.777470:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176182.777474:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176182.777477:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.777479:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176182.777482:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176182.777486:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.777491:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176182.777494:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786176182.777499:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176182.777503:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176182.777506:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176182.777509:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176182.777534:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176182.777536:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176182.777539:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:2.0:1786176182.777543:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176182.777551:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856222464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176182.777585:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176182.777588:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176182.777594:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856222464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176182.777607:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176182.777613:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6416)
00000800:00000200:0.1:1786176182.777718:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176182.777720:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176182.777908:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176182.777909:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176182.778087:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176182.778089:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176182.778266:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176182.778268:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:2.0:1786176183.352818:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176183.352823:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176183.352844:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176184.376699:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176184.376704:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176184.376709:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.0:1786176184.696673:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176184.696682:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176184.696687:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000020:00000001:2.0:1786176185.400694:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176185.400698:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176185.400701:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176186.424644:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176186.424648:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176186.424650:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176187.448752:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176187.448758:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176187.448762:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176187.896591:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176187.896600:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176187.896602:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176187.896606:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176187.896612:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176187.896621:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6411
00000400:00000200:2.0:1786176187.896630:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176187.896638:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176187.896642:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176187.896648:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.896651:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176187.896655:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.896657:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176187.896660:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176187.896661:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176187.896663:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176187.896665:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176187.896667:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176187.896669:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176187.896672:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.896674:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176187.896675:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.896676:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176187.896679:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176187.896681:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176187.896682:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176187.896685:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176187.896687:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176187.896689:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.896690:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176187.896691:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176187.896698:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000020:00000040:2.0:1786176187.896701:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176187.896704:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176187.896705:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.896707:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176187.896709:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176187.896712:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176187.896718:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786176187.896723:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176187.896730:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856222592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176187.896738:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176187.896742:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176187.896745:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856222592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176187.896784:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176187.896789:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176187.896793:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176187.896795:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176187.896797:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176187.896801:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176187.896805:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6411
00000400:00000200:2.0:1786176187.896810:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176187.896815:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176187.896819:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176187.896824:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.896828:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176187.896833:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.896836:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176187.896840:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176187.896846:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176187.896848:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176187.896853:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176187.896856:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176187.896860:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176187.896867:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.896873:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176187.896879:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.896884:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176187.896890:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176187.896896:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176187.896900:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176187.896905:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176187.896911:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176187.896917:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.896922:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176187.896928:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176187.896936:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.896944:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176187.896954:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000020:00000040:2.0:1786176187.896961:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176187.896969:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176187.896974:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.896980:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176187.896983:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176187.896990:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176187.896997:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786176187.897005:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176187.897017:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856222720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176187.897038:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176187.897055:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176187.897063:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856222720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176187.897078:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176187.897084:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176187.897094:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176187.897100:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176187.897106:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176187.897131:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176187.897139:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6411
00000400:00000200:2.0:1786176187.897149:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176187.897158:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176187.897162:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176187.897167:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897168:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176187.897173:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897175:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176187.897180:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176187.897181:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176187.897184:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176187.897188:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000800:00000200:0.1:1786176187.897191:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786176187.897192:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176187.897198:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:0.1:1786176187.897198:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:2.0:1786176187.897203:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897208:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176187.897212:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897215:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176187.897217:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176187.897224:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897227:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176187.897230:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176187.897233:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176187.897237:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.897239:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176187.897241:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176187.897245:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.897247:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176187.897254:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000020:00000040:2.0:1786176187.897258:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176187.897261:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176187.897263:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.897265:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176187.897275:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176187.897277:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176187.897283:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:2.0:1786176187.897287:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176187.897297:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856222848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176187.897332:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176187.897338:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176187.897347:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856222848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176187.897371:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176187.897384:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176187.897392:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176187.897395:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176187.897398:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176187.897403:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176187.897409:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6411
00000400:00000200:2.0:1786176187.897420:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176187.897425:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176187.897431:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176187.897438:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897444:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176187.897450:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897453:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176187.897459:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176187.897463:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176187.897483:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176187.897487:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176187.897489:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176187.897492:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176187.897499:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897503:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176187.897507:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897511:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176187.897546:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176187.897549:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897551:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176187.897552:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176187.897554:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176187.897558:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.897560:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176187.897563:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176187.897567:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.897570:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176187.897575:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000020:00000040:2.0:1786176187.897579:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176187.897584:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176187.897587:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.897591:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176187.897593:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176187.897595:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176187.897601:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786176187.897607:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176187.897616:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856222976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176187.897630:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176187.897634:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176187.897641:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856222976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176187.897648:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176187.897650:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176187.897653:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176187.897656:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176187.897658:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176187.897664:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176187.897669:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6411
00000400:00000200:2.0:1786176187.897676:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176187.897681:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176187.897685:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176187.897691:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897696:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176187.897700:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897703:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176187.897706:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176187.897708:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176187.897713:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176187.897715:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176187.897719:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176187.897722:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176187.897728:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897732:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176187.897734:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897737:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176187.897740:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786176187.897744:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176187.897749:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176187.897773:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176187.897777:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176187.897782:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.897785:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176187.897788:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176187.897794:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.897800:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176187.897808:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000020:00000040:2.0:1786176187.897815:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176187.897822:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176187.897826:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176187.897833:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176187.897837:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176187.897841:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176187.897847:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786176187.897853:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176187.897865:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856223104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176187.897886:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176187.897892:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176187.897900:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856223104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176187.897920:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176187.897926:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6421)
00000800:00000200:0.1:1786176187.898009:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176187.898012:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176187.898248:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176187.898250:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176187.898552:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176187.898555:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176187.899071:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176187.899074:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:2.0:1786176189.496746:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176189.496750:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176189.496754:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176190.520844:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176190.520850:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176190.520855:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176192.568733:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176192.568749:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176192.568754:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176193.017676:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176193.017681:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176193.018740:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176193.018743:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176193.019226:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176193.019231:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176193.019603:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176193.019605:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176193.020115:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176193.020118:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:2.0:1786176193.592713:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176193.592718:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176193.592722:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176195.640747:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176195.640757:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176195.640764:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176196.664765:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176196.664774:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176196.664781:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176198.137811:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176198.137822:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176198.139153:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176198.139157:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176198.139752:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176198.139756:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176198.140273:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176198.140279:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176198.140876:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176198.140880:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:2.0:1786176198.712828:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176198.712837:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176198.712844:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176199.736766:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176199.736773:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176199.736778:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176201.784769:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176201.784778:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176201.784785:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176202.808734:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176202.808739:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176202.808742:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176203.257258:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176203.257741:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176203.257901:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176203.258052:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176203.258219:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786176204.856734:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176204.856740:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176204.856744:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176205.880657:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176205.880661:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176205.880663:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:2.0:1786176206.328883:0:2379:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:2.0:1786176206.328952:0:2379:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1786176206.328981:0:2379:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176206.328987:0:2379:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:2.0:1786176206.328994:0:2379:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176206.329000:0:2379:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1786176206.329006:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176206.329011:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:2.0:1786176206.329018:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176206.329026:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176206.329037:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:2.0:1786176206.329042:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:2.0:1786176206.329051:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176206.329082:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa7480 x1872934856224512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176206.329116:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176206.329123:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7480 x1872934856224512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176206.329136:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176206.329139:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176206.329154:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176206.329159:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45200.
00000400:00000200:2.0:1786176206.329164:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176206.329170:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176206.329175:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4990
00000400:00000010:2.0:1786176206.329178:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4990.
00000100:00000001:2.0:1786176206.329182:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176206.329191:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa6680 x1872934856224640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176206.329211:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176206.329221:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa6680 x1872934856224640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176206.329234:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176206.329241:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176206.329258:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176206.329268:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45a00.
00000400:00000200:2.0:1786176206.329277:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176206.329283:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176206.329313:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:2.0:1786176206.329322:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:2.0:1786176206.329331:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176206.329384:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa5c00 x1872934856224768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176206.329438:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176206.329448:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5c00 x1872934856224768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176206.329471:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176206.329476:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176206.329494:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176206.329505:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:2.0:1786176206.329589:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176206.329610:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176206.329621:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:2.0:1786176206.329628:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:2.0:1786176206.329638:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176206.329651:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa4000 x1872934856224896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176206.329676:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176206.329689:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4000 x1872934856224896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176206.329732:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176206.329742:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176206.329758:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176206.329764:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:2.0:1786176206.329769:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176206.329778:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176206.329782:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4a18
00000400:00000010:2.0:1786176206.329787:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4a18.
00000100:00000001:2.0:1786176206.329797:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176206.329810:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa4700 x1872934856225024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176206.329868:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176206.329879:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4700 x1872934856225024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176206.329900:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176206.329908:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:2.0:1786176207.928680:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176207.928686:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176207.928690:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176208.377021:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176208.377026:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176208.377579:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176208.377581:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176208.377909:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176208.377912:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176208.378217:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176208.378219:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176208.378505:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176208.378507:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:2.0:1786176208.952677:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176208.952680:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176208.952682:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176211.000790:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176211.000799:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176211.000806:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176212.024778:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176212.024784:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176212.024788:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176213.497740:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176213.497756:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786176213.499136:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176213.499140:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786176213.499595:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176213.499599:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786176213.500041:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176213.500047:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786176213.500671:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176213.500675:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:2.0:1786176214.072687:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176214.072691:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176214.072694:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176215.096773:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176215.096782:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176215.096789:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176217.144873:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176217.144880:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176217.144886:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176218.168796:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176218.168805:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176218.168811:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176218.617650:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176218.617662:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786176218.618980:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176218.618983:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786176218.619401:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176218.619405:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786176218.619886:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176218.619892:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786176218.620440:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176218.620443:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000020:00000001:2.0:1786176220.216732:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176220.216737:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176220.216740:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176221.240733:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176221.240740:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176221.240746:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.0:1786176222.584622:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176222.584630:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176222.584635:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000020:00000001:2.0:1786176223.288706:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176223.288710:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176223.288713:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176223.737586:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176223.737598:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786176223.739150:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176223.739153:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786176223.739435:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176223.739437:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786176223.739686:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176223.739688:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786176223.740036:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176223.740038:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:2.0:1786176224.312748:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176224.312759:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176224.312769:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176227.384728:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176227.384752:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176227.384755:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176228.857784:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176228.857796:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176228.858961:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176228.858963:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176228.859272:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176228.859274:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176228.859459:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176228.859460:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176228.859768:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176228.859770:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:2.0:1786176230.456863:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176230.456870:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176230.456874:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176233.528802:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176233.528808:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176233.528813:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176233.978155:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176233.978167:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176233.980160:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176233.980163:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176233.980682:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176233.980686:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176233.981149:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176233.981153:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176233.981855:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176233.981858:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:2.0:1786176236.600866:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176236.600871:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176236.600875:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176239.097409:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176239.097430:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176239.098283:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176239.098285:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176239.098640:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176239.098643:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176239.098885:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176239.098887:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176239.099174:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176239.099176:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:2.0:1786176239.672813:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176239.672817:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176239.672820:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176242.744674:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176242.744680:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176242.744684:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176244.216642:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176244.216648:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176244.216650:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176244.216653:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176244.216657:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176244.216663:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6467
00000400:00000200:2.0:1786176244.216670:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176244.216675:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176244.216677:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176244.216680:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.216683:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176244.216685:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.216686:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176244.216689:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176244.216691:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176244.216693:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176244.216695:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176244.216697:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176244.216698:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176244.216700:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.216702:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176244.216703:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.216704:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176244.216706:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176244.216708:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176244.216710:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176244.216712:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176244.216714:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176244.216716:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.216718:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176244.216719:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176244.216725:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786176244.216728:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176244.216731:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176244.216732:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.216734:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176244.216736:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176244.216740:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176244.216746:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786176244.216752:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176244.216759:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856229632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176244.216767:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176244.216771:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176244.216774:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856229632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176244.216791:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176244.216794:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176244.216796:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176244.216797:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176244.216799:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176244.216801:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176244.216803:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6467
00000400:00000200:2.0:1786176244.216805:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176244.216807:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176244.216809:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176244.216812:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.216815:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176244.216819:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.216822:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176244.216824:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176244.216827:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176244.216828:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176244.216830:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176244.216833:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176244.216837:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176244.216844:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.216849:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176244.216852:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.216855:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176244.216858:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176244.216861:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176244.216865:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176244.216868:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176244.216875:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176244.216880:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.216882:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176244.216886:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176244.216890:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.216893:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176244.216897:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176244.216900:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176244.216903:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176244.216906:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.216910:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176244.216911:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176244.216913:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176244.216918:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786176244.216922:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176244.216929:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856229760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176244.216938:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176244.216941:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176244.216947:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856229760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176244.216958:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176244.216962:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176244.216983:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176244.216986:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176244.216988:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176244.216994:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176244.216999:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6467
00000400:00000200:2.0:1786176244.217005:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176244.217010:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176244.217012:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176244.217018:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217020:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176244.217022:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217024:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176244.217029:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176244.217031:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176244.217034:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176244.217035:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176244.217039:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176244.217044:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176244.217048:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217051:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176244.217055:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217059:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176244.217063:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176244.217069:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217075:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176244.217079:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176244.217082:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176244.217088:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.217091:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176244.217093:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176244.217095:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.217106:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176244.217113:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786176244.217116:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176244.217120:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176244.217121:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.217123:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176244.217125:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176244.217127:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176244.217135:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786176244.217142:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176244.217147:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:2.0:1786176244.217149:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856229888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1786176244.217154:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1786176244.217157:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176244.217161:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176244.217165:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856229888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176244.217175:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176244.217180:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176244.217188:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176244.217191:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176244.217193:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176244.217196:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176244.217201:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6467
00000400:00000200:2.0:1786176244.217206:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176244.217211:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176244.217214:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176244.217220:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217223:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176244.217227:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217231:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176244.217235:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176244.217238:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176244.217239:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176244.217241:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176244.217242:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176244.217243:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176244.217245:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217248:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176244.217251:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217252:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176244.217256:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176244.217265:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217271:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176244.217274:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176244.217279:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176244.217285:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.217288:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176244.217290:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176244.217292:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.217294:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176244.217297:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786176244.217302:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176244.217306:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176244.217308:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.217311:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176244.217312:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176244.217313:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176244.217317:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786176244.217320:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176244.217326:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856230016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176244.217332:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176244.217333:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176244.217339:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856230016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176244.217348:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176244.217352:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176244.217359:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176244.217362:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176244.217365:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176244.217368:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176244.217372:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6467
00000400:00000200:2.0:1786176244.217382:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176244.217390:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176244.217394:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176244.217399:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217403:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176244.217410:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217417:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176244.217422:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176244.217425:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176244.217429:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176244.217432:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176244.217436:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176244.217440:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176244.217446:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217449:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176244.217454:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217456:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176244.217459:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786176244.217465:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176244.217470:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176244.217474:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176244.217478:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176244.217481:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.217484:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176244.217488:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176244.217494:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.217498:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176244.217504:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786176244.217530:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176244.217534:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176244.217537:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176244.217540:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176244.217544:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176244.217547:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176244.217554:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786176244.217560:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176244.217567:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856230144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176244.217586:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176244.217590:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176244.217596:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856230144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176244.217610:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176244.217616:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6477)
00000800:00000200:3.1:1786176244.217743:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176244.217745:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786176244.217926:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176244.217927:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786176244.218141:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176244.218144:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786176244.218373:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176244.218375:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:2.0:1786176245.816801:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176245.816807:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176245.816811:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176248.888652:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176248.888656:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176248.888659:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176249.337230:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176249.337238:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176249.337883:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176249.337885:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176249.338107:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176249.338109:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176249.338322:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176249.338324:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176249.338530:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176249.338532:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:2.0:1786176251.960740:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176251.960746:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176251.960751:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176254.457286:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176254.457292:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176254.458020:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176254.458023:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176254.458268:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176254.458270:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176254.458535:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176254.458537:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176254.458737:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176254.458739:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:2.0:1786176255.032768:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176255.032772:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176255.032774:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176258.104771:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176258.104777:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176258.104782:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176259.576655:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176259.576668:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176259.576672:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176259.576681:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176259.576691:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176259.576700:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6482
00000400:00000200:2.0:1786176259.576714:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176259.576723:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176259.576727:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176259.576732:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.576736:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176259.576740:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.576744:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176259.576749:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176259.576752:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176259.576755:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176259.576774:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176259.576776:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176259.576778:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176259.576780:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.576781:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176259.576782:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.576784:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176259.576786:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176259.576788:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176259.576790:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176259.576793:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176259.576795:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176259.576797:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.576799:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176259.576801:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176259.576809:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786176259.576811:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176259.576815:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176259.576816:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.576818:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176259.576820:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176259.576848:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176259.576855:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:2.0:1786176259.576861:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176259.576869:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856231552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176259.576877:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176259.576880:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176259.576884:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856231552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176259.576899:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176259.576901:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176259.576904:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176259.576905:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176259.576906:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176259.576908:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176259.576910:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6482
00000400:00000200:2.0:1786176259.576913:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176259.576915:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176259.576917:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176259.576919:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.576920:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176259.576922:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.576924:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176259.576927:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176259.576928:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176259.576929:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176259.576931:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176259.576932:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176259.576934:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176259.576935:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.576937:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176259.576938:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.576940:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176259.576942:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176259.576944:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176259.576947:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176259.576949:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176259.576954:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176259.576955:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.576957:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176259.576960:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176259.576968:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.576975:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176259.576986:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786176259.576990:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176259.576992:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176259.576993:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.576994:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176259.576998:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176259.576999:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176259.577003:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786176259.577010:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176259.577028:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856231680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176259.577052:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176259.577063:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176259.577079:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856231680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176259.577097:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176259.577109:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176259.577122:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176259.577125:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176259.577128:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176259.577131:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176259.577137:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6482
00000400:00000200:2.0:1786176259.577142:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176259.577155:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176259.577161:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176259.577172:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577177:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176259.577187:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577197:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176259.577207:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176259.577211:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176259.577217:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176259.577221:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176259.577227:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176259.577233:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176259.577237:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577239:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176259.577240:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577241:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176259.577243:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176259.577245:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577246:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176259.577253:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176259.577263:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176259.577292:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.577295:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176259.577297:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176259.577305:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.577309:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176259.577313:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000020:00000040:2.0:1786176259.577317:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176259.577325:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176259.577327:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.577333:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176259.577339:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176259.577345:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176259.577357:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786176259.577368:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176259.577398:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856231808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176259.577419:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176259.577423:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176259.577432:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856231808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176259.577452:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176259.577465:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1786176259.577473:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786176259.577475:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176259.577480:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.1:1786176259.577482:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000100:00080000:2.0:1786176259.577484:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176259.577491:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176259.577498:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6482
00000400:00000200:2.0:1786176259.577506:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176259.577510:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176259.577511:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176259.577513:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577516:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176259.577519:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577524:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176259.577526:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176259.577526:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176259.577527:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176259.577529:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176259.577530:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176259.577531:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176259.577533:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577535:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176259.577537:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577539:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176259.577540:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176259.577578:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577585:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176259.577589:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176259.577595:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176259.577604:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.577614:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176259.577619:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176259.577645:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.577652:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176259.577659:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000020:00000040:2.0:1786176259.577664:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176259.577667:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176259.577670:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.577677:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176259.577678:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176259.577680:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176259.577686:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786176259.577699:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176259.577709:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856231936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176259.577717:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176259.577722:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176259.577725:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856231936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176259.577735:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176259.577738:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176259.577740:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176259.577742:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176259.577744:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176259.577751:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176259.577754:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6482
00000400:00000200:2.0:1786176259.577761:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176259.577767:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176259.577771:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176259.577801:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577814:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176259.577839:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577844:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176259.577850:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176259.577856:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176259.577877:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176259.577882:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176259.577888:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176259.577900:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176259.577908:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577915:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176259.577930:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577940:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176259.577950:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786176259.577960:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176259.577966:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176259.577973:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176259.577979:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176259.577987:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.577991:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176259.578000:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176259.578008:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.578010:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176259.578017:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786176259.578024:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176259.578033:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176259.578035:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176259.578047:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176259.578050:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176259.578055:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176259.578065:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786176259.578070:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176259.578083:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856232064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176259.578101:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176259.578108:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176259.578118:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856232064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176259.578142:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176259.578154:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6493)
00000800:00000200:0.1:1786176259.578336:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176259.578338:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786176259.578571:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176259.578573:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786176259.578889:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176259.578901:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786176259.579096:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176259.579098:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:2.0:1786176261.176877:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176261.176884:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176261.176889:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.0:1786176261.496609:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176261.496616:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176261.496620:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000020:00000001:2.0:1786176264.248864:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176264.248875:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176264.248883:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176264.696646:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176264.696654:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176264.696659:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176264.696667:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176264.696675:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176264.696680:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176264.696685:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176264.696691:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176264.696709:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176264.697688:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176264.697709:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176264.698815:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176264.698818:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176264.699266:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176264.699269:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176264.699532:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176264.699535:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176264.699936:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176264.699939:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:2.0:1786176265.720607:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176265.720614:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176265.720616:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176265.720622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176265.720625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176265.720629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176265.720631:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176265.720633:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176265.720636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176266.744594:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176266.744601:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176266.744602:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176266.744607:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176266.744609:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176266.744613:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176266.744615:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176266.744618:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176266.744620:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176267.320877:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176267.320911:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176267.320915:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176267.768598:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176267.768606:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176267.768608:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176267.768614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176267.768618:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176267.768621:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176267.768624:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176267.768627:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176267.768653:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176268.792639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176268.792647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176268.792649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176268.792654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176268.792658:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176268.792661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176268.792664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176268.792667:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176268.792669:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176269.816624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176269.816633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176269.816637:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176269.816645:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176269.816651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176269.816656:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176269.816660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176269.816666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176269.816671:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176269.817359:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176269.817939:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176269.818434:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176269.818808:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176269.819001:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786176270.392751:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176270.392755:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176270.392758:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176270.840639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176270.840649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176270.840651:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176270.840657:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176270.840672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176270.840678:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176270.840682:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176270.840685:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176270.840688:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176271.864580:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176271.864593:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176271.864595:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176271.864600:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176271.864604:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176271.864608:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176271.864610:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176271.864613:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176271.864616:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176272.888609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176272.888623:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176272.888626:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176272.888635:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176272.888641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176272.888646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176272.888650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176272.888655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176272.888659:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176273.464776:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176273.464785:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176273.464792:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176273.912597:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176273.912608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176273.912612:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176273.912618:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176273.912623:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176273.912628:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176273.912632:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176273.912636:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176273.912640:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176274.936716:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176274.936727:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176274.936737:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176274.936748:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176274.936767:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176274.936782:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176274.936798:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176274.936809:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176274.936820:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176274.937998:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176274.938010:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176274.939097:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176274.939100:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176274.939582:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176274.939585:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176274.939871:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176274.939874:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176274.940170:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176274.940173:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1786176275.960655:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176275.960684:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176275.960687:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176275.960695:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176275.960699:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176275.960704:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176275.960708:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176275.960711:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176275.960715:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176276.536743:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176276.536746:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176276.536749:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176276.984672:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176276.984700:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176276.984702:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176276.984709:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176276.984713:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176276.984716:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176276.984720:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176276.984723:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176276.984725:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176278.008714:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176278.008722:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176278.008724:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176278.008730:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176278.008733:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176278.008736:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176278.008739:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176278.008742:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176278.008744:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176279.032752:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176279.032764:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176279.032769:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176279.032780:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176279.032786:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176279.032793:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176279.032799:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176279.032804:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176279.032810:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176279.608672:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176279.608677:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176279.608680:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176280.056727:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176280.056741:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176280.056749:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176280.056761:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176280.056773:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176280.056785:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176280.056793:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176280.056802:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176280.056814:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176280.057857:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176280.057869:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176280.058837:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176280.058840:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176280.059196:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176280.059199:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176280.059533:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176280.059538:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176280.059894:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176280.059896:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1786176281.080611:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176281.080638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176281.080641:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176281.080647:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176281.080651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176281.080655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176281.080674:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176281.080677:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176281.080681:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176282.104595:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176282.104601:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176282.104624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176282.104628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176282.104631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176282.104634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176282.104637:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176282.104639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176282.104642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176282.680775:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176282.680780:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176282.680785:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176283.128657:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176283.128664:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176283.128665:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176283.128670:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176283.128672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176283.128676:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176283.128678:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176283.128680:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176283.128682:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176284.152644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176284.152652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176284.152655:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176284.152661:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176284.152664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176284.152668:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176284.152671:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176284.152674:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176284.152677:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176285.176644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176285.176651:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176285.176653:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176285.176658:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176285.176663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176285.176666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176285.176669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176285.176672:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176285.176675:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176285.177244:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176285.177252:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176285.177889:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176285.177890:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176285.178079:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176285.178080:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176285.178278:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176285.178280:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176285.178558:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176285.178560:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:2.0:1786176285.752780:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176285.752786:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176285.752789:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176286.200668:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176286.200695:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176286.200699:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176286.200709:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176286.200715:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176286.200721:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176286.200726:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176286.200733:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176286.200737:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176287.224655:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176287.224664:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176287.224667:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176287.224673:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176287.224677:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176287.224681:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176287.224684:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176287.224687:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176287.224690:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176288.248654:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176288.248660:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176288.248662:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176288.248666:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176288.248669:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176288.248672:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176288.248675:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176288.248678:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176288.248681:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176288.824745:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176288.824752:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176288.824757:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176289.272617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176289.272624:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176289.272625:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176289.272630:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176289.272633:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176289.272636:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176289.272638:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176289.272640:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176289.272642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176290.296667:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176290.296673:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176290.296675:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176290.296681:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176290.296685:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176290.296689:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176290.296694:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176290.296697:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176290.296699:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176290.297348:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176290.297354:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176290.297981:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176290.297983:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176290.298208:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176290.298210:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176290.298442:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176290.298444:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176290.298670:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176290.298672:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1786176291.320638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176291.320645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176291.320649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176291.320653:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176291.320656:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176291.320661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176291.320663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176291.320666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176291.320668:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176291.896669:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176291.896674:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176291.896685:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176292.344662:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176292.344670:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176292.344672:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176292.344677:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176292.344680:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176292.344684:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176292.344687:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176292.344689:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176292.344692:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176293.368655:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176293.368666:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176293.368669:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176293.368678:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176293.368684:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176293.368690:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176293.368695:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176293.368700:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176293.368704:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176294.392690:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176294.392703:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176294.392706:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176294.392715:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176294.392740:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176294.392748:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176294.392753:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176294.392758:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176294.392762:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176294.968758:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176294.968763:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176294.968767:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176295.416613:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176295.416620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176295.416624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176295.416631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176295.416637:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176295.416642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176295.416646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176295.416649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176295.416655:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176295.417337:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176295.417347:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786176295.418031:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176295.418033:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786176295.418263:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176295.418265:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786176295.418468:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176295.418470:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786176295.418719:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176295.418737:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:2.0:1786176296.440666:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176296.440678:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176296.440681:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176296.440689:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176296.440694:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176296.440699:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176296.440703:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176296.440707:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176296.440711:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176297.464582:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176297.464587:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176297.464589:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176297.464593:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176297.464596:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176297.464599:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176297.464601:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176297.464603:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176297.464605:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176298.040677:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176298.040681:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176298.040684:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176298.488643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176298.488651:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176298.488653:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176298.488694:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176298.488698:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176298.488702:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176298.488705:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176298.488708:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176298.488712:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176299.512593:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176299.512601:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176299.512604:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176299.512611:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176299.512615:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176299.512619:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176299.512623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176299.512626:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176299.512629:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786176300.408656:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176300.408688:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176300.408695:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:2.0:1786176300.536663:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176300.536674:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176300.536682:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176300.536691:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176300.536701:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176300.536707:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176300.536716:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176300.536721:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176300.536738:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176300.537732:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176300.537739:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176300.538908:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176300.538911:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176300.539257:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176300.539261:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176300.539710:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176300.539724:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176300.540160:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176300.540163:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000020:00000001:2.0:1786176301.112759:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176301.112767:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176301.112775:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176301.560629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176301.560638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176301.560640:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176301.560646:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176301.560649:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176301.560653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176301.560655:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176301.560658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176301.560660:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176302.584665:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176302.584673:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176302.584675:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176302.584681:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176302.584685:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176302.584688:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176302.584691:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176302.584694:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176302.584697:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176303.608598:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176303.608606:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176303.608609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176303.608615:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176303.608620:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176303.608624:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176303.608628:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176303.608631:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176303.608634:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176304.184748:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176304.184754:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176304.184759:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176304.632634:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176304.632642:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176304.632645:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176304.632650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176304.632654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176304.632658:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176304.632661:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176304.632664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176304.632667:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176305.656586:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176305.656592:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176305.656594:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176305.656599:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176305.656606:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176305.656613:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176305.656620:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176305.656626:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176305.656630:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176305.657336:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176305.657345:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176305.658135:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176305.658141:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176305.658491:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176305.658493:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176305.658779:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176305.658781:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176305.659027:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176305.659029:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:2.0:1786176306.680580:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176306.680586:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176306.680588:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176306.680592:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176306.680595:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176306.680598:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176306.680600:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176306.680602:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176306.680604:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176307.704596:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176307.704620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176307.704623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176307.704629:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176307.704633:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176307.704637:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176307.704640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176307.704644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176307.704647:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176308.728579:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176308.728586:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176308.728588:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176308.728592:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176308.728595:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176308.728598:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176308.728600:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176308.728602:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176308.728604:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176309.752644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176309.752653:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176309.752655:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176309.752661:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176309.752665:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176309.752669:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176309.752673:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176309.752676:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176309.752679:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176310.776638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176310.776649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176310.776656:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176310.776666:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176310.776675:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176310.776680:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176310.776683:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176310.776687:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176310.776690:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176310.777447:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176310.777453:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176310.778300:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176310.778303:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176310.778655:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176310.778658:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176310.779008:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176310.779012:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176310.779358:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176310.779361:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1786176311.800591:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176311.800597:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176311.800599:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176311.800603:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176311.800605:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176311.800609:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176311.800612:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176311.800616:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176311.800618:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176312.824580:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176312.824589:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176312.824591:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176312.824597:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176312.824601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176312.824604:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176312.824606:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176312.824609:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176312.824611:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176313.848628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176313.848637:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176313.848639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176313.848647:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176313.848653:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176313.848657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176313.848660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176313.848664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176313.848668:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176314.872612:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176314.872621:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176314.872624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176314.872631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176314.872635:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176314.872639:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176314.872642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176314.872645:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176314.872648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176315.896585:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176315.896594:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176315.896597:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176315.896603:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176315.896608:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176315.896612:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176315.896616:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176315.896620:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176315.896623:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176315.897552:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176315.897556:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176315.898612:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176315.898618:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176315.899350:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176315.899351:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176315.899807:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176315.899810:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176315.900081:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176315.900083:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1786176316.920601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176316.920611:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176316.920613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176316.920621:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176316.920626:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176316.920630:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176316.920635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176316.920639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176316.920643:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176317.944673:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176317.944683:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176317.944685:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176317.944690:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176317.944694:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176317.944698:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176317.944701:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176317.944703:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176317.944706:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176318.968572:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176318.968579:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176318.968582:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176318.968587:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176318.968591:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176318.968595:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176318.968598:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176318.968600:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176318.968603:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176319.992617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176319.992625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176319.992627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176319.992633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176319.992637:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176319.992640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176319.992642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176319.992645:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176319.992648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176321.016627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176321.016635:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176321.016640:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176321.016648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176321.016654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176321.016658:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176321.016662:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176321.016668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176321.016677:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176321.017424:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176321.017433:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176321.018425:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176321.018428:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176321.018817:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176321.018822:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176321.019200:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176321.019202:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176321.019655:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176321.019659:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1786176322.040666:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176322.040673:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176322.040675:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176322.040682:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176322.040688:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176322.040694:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176322.040698:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176322.040701:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176322.040704:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176323.064613:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176323.064622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176323.064625:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176323.064631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176323.064635:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176323.064639:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176323.064642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176323.064645:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176323.064648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176324.088647:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176324.088658:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176324.088660:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176324.088667:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176324.088672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176324.088676:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176324.088680:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176324.088684:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176324.088687:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176325.112579:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176325.112586:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176325.112588:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176325.112593:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176325.112596:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176325.112598:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176325.112601:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176325.112603:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176325.112605:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176326.136618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176326.136626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176326.136630:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176326.136636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176326.136644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176326.136651:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176326.136659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176326.136666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176326.136673:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176326.137358:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176326.137363:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176326.138042:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176326.138044:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176326.138314:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176326.138315:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176326.138564:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176326.138566:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176326.138814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176326.138817:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:2.0:1786176327.160676:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176327.160685:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176327.160687:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176327.160693:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176327.160698:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176327.160702:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176327.160706:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176327.160708:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176327.160711:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176328.184597:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176328.184605:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176328.184607:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176328.184613:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176328.184617:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176328.184621:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176328.184625:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176328.184628:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176328.184631:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176329.208642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176329.208651:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176329.208654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176329.208660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176329.208664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176329.208668:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176329.208672:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176329.208675:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176329.208678:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176330.232669:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176330.232676:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176330.232678:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176330.232682:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176330.232686:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176330.232689:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176330.232691:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176330.232693:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176330.232696:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176331.256674:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176331.256684:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176331.256695:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176331.256722:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176331.256729:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176331.256738:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176331.256756:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176331.256760:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176331.256768:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176331.257442:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176331.257451:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176331.258302:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176331.258303:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176331.258620:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176331.258622:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176331.258917:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176331.258920:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176331.259232:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176331.259234:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1786176332.280588:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176332.280597:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176332.280599:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176332.280605:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176332.280609:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176332.280613:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176332.280616:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176332.280619:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176332.280622:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176333.304638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176333.304645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176333.304647:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176333.304651:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176333.304655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176333.304659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176333.304661:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176333.304663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176333.304665:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176334.328607:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176334.328616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176334.328633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176334.328640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176334.328645:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176334.328650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176334.328654:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176334.328657:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176334.328660:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176335.352656:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176335.352666:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176335.352668:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176335.352675:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176335.352680:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176335.352684:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176335.352688:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176335.352692:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176335.352696:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176336.376570:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176336.376576:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176336.376578:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176336.376583:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176336.376587:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176336.376589:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176336.376593:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176336.376595:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176336.376597:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176336.377172:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176336.377631:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176336.377813:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176336.377969:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176336.378132:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176337.400607:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176337.400614:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176337.400615:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176337.400620:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176337.400623:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176337.400626:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176337.400629:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176337.400631:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176337.400634:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786176338.296706:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176338.296714:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176338.296719:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:2.0:1786176338.424636:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176338.424645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176338.424648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176338.424654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176338.424678:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176338.424684:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176338.424691:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176338.424696:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176338.424701:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176339.448618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176339.448628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176339.448630:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176339.448636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176339.448668:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176339.448673:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176339.448677:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176339.448680:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176339.448683:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176340.472647:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176340.472655:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176340.472658:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176340.472664:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176340.472668:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176340.472672:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176340.472675:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176340.472678:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176340.472681:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176341.496644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176341.496656:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176341.496661:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176341.496671:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176341.496677:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176341.496683:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176341.496687:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176341.496692:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176341.496706:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176341.497447:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176341.497452:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176341.498280:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176341.498282:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176341.498584:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176341.498587:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176341.498816:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176341.498827:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176341.499090:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176341.499092:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1786176342.520569:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176342.520576:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176342.520578:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176342.520582:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176342.520586:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176342.520588:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176342.520591:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176342.520594:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176342.520597:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176343.544626:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176343.544632:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176343.544634:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176343.544638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176343.544641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176343.544644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176343.544646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176343.544649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176343.544651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176344.568613:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176344.568620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176344.568621:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176344.568627:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176344.568630:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176344.568632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176344.568634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176344.568637:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176344.568639:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176345.592615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176345.592623:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176345.592625:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176345.592630:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176345.592633:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176345.592637:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176345.592640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176345.592643:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176345.592645:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176346.616622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176346.616629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176346.616631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176346.616636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176346.616641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176346.616645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176346.616649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176346.616654:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176346.616657:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176346.617215:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176346.617218:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1786176346.617932:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176346.617936:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1786176346.618185:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176346.618187:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1786176346.618680:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176346.618683:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1786176346.619109:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176346.619113:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000100:00000001:2.0:1786176347.640602:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176347.640607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176347.640609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176347.640614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176347.640617:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176347.640620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176347.640622:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176347.640624:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176347.640627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176348.664612:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176348.664619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176348.664622:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176348.664626:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176348.664629:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176348.664632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176348.664634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176348.664636:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176348.664638:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176349.688619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176349.688625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176349.688627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176349.688643:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176349.688647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176349.688651:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176349.688654:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176349.688657:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176349.688660:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176350.712624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176350.712631:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176350.712633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176350.712637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176350.712640:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176350.712643:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176350.712645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176350.712647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176350.712649:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176351.736573:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176351.736580:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176351.736582:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176351.736586:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176351.736590:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176351.736593:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176351.736597:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176351.736601:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176351.736605:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176351.737120:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176351.737127:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1786176351.737848:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176351.737853:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1786176351.738122:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176351.738124:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1786176351.738307:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176351.738309:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1786176351.738578:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176351.738580:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:2.0:1786176352.760585:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176352.760592:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176352.760594:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176352.760599:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176352.760603:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176352.760606:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176352.760609:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176352.760611:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176352.760613:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176353.784610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176353.784617:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176353.784619:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176353.784624:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176353.784627:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176353.784631:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176353.784634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176353.784636:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176353.784639:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176354.808747:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176354.808756:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176354.808759:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176354.808766:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176354.808771:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176354.808776:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176354.808780:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176354.808784:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176354.808788:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176355.832631:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176355.832639:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176355.832642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176355.832648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176355.832653:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176355.832657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176355.832661:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176355.832664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176355.832667:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176356.856656:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176356.856664:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176356.856667:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176356.856675:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176356.856684:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176356.856691:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176356.856699:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176356.856704:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176356.856713:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176356.857556:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176356.857564:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786176356.858665:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176356.858672:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786176356.859609:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176356.859612:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786176356.860177:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176356.860182:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786176356.860568:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176356.860571:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:2.0:1786176361.976752:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176361.976762:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176361.976769:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176361.976778:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176361.976787:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176361.976796:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176361.976804:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176361.976810:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176361.976819:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176361.977339:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176361.977346:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1786176361.978070:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176361.978072:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1786176361.978259:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176361.978260:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1786176361.978433:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176361.978435:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1786176361.978711:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176361.978713:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:2.0:1786176363.000576:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176363.000582:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176363.000584:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176363.000589:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176363.000591:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176363.000594:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176363.000597:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176363.000599:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176363.000601:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176364.024564:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176364.024570:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176364.024571:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176364.024575:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176364.024578:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176364.024580:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176364.024583:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176364.024585:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176364.024588:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176365.048603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176365.048611:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176365.048613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176365.048618:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176365.048622:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176365.048625:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176365.048628:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176365.048631:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176365.048635:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176366.072599:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176366.072608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176366.072611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176366.072617:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176366.072622:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176366.072626:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176366.072629:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176366.072632:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176366.072636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176367.096589:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176367.096595:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176367.096599:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176367.096609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176367.096619:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176367.096624:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176367.096627:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176367.096630:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176367.096636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176367.097342:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176367.097360:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786176367.098010:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176367.098013:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786176367.098332:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176367.098335:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786176367.098636:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176367.098639:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786176367.098928:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176367.098930:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1786176368.120601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176368.120608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176368.120610:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176368.120615:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176368.120618:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176368.120621:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176368.120624:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176368.120626:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176368.120628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176369.144596:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176369.144608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176369.144612:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176369.144621:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176369.144626:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176369.144630:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176369.144633:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176369.144635:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176369.144639:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176370.168625:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176370.168634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176370.168636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176370.168641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176370.168645:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176370.168649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176370.168652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176370.168654:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176370.168657:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176371.192631:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176371.192641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176371.192643:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176371.192650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176371.192654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176371.192658:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176371.192661:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176371.192664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176371.192667:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176372.216649:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176372.216656:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176372.216662:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176372.216670:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176372.216678:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176372.216700:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176372.216710:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176372.216716:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176372.216723:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176372.217620:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176372.217628:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176372.219086:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176372.219089:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176372.219551:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176372.219555:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176372.219960:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176372.219964:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176372.220503:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176372.220507:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:2.0:1786176373.240581:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176373.240587:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176373.240589:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176373.240593:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176373.240595:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176373.240598:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176373.240600:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176373.240603:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176373.240606:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176374.264688:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176374.264701:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176374.264706:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176374.264737:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176374.264743:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176374.264749:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176374.264753:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176374.264775:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176374.264779:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176375.288585:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176375.288592:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176375.288594:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176375.288599:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176375.288603:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176375.288607:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176375.288609:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176375.288612:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176375.288614:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176376.312621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176376.312652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176376.312655:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176376.312662:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176376.312667:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176376.312671:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176376.312675:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176376.312678:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176376.312682:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176376.888824:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176376.888828:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176376.888832:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.0:1786176377.208679:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176377.208706:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176377.208713:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:2.0:1786176377.336704:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176377.336734:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176377.336741:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176377.336756:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176377.336768:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176377.336783:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176377.336791:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176377.336799:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176377.336808:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176377.337317:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176377.337323:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176377.337959:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176377.337961:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176377.338153:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176377.338155:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176377.338336:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176377.338338:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176377.338539:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176377.338541:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1786176378.360645:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176378.360653:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176378.360655:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176378.360660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176378.360664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176378.360668:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176378.360671:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176378.360690:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176378.360694:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176379.384594:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176379.384602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176379.384604:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176379.384610:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176379.384614:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176379.384617:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176379.384620:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176379.384623:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176379.384625:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176379.960738:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176379.960744:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176379.960749:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176380.408627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176380.408635:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176380.408637:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176380.408644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176380.408648:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176380.408651:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176380.408665:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176380.408668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176380.408672:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176381.432580:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176381.432587:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176381.432589:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176381.432594:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176381.432597:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176381.432600:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176381.432603:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176381.432606:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176381.432608:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176382.456631:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176382.456710:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176382.456716:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176382.456730:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176382.456742:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176382.456753:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176382.456778:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176382.456787:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176382.456811:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176382.457895:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176382.457906:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176382.459134:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176382.459139:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176382.459537:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176382.459540:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176382.460002:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176382.460005:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176382.460424:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176382.460430:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:2.0:1786176383.032715:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176383.032719:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176383.032722:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176383.480712:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176383.480719:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176383.480722:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176383.480728:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176383.480732:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176383.480736:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176383.480740:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176383.480743:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176383.480746:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176384.504596:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176384.504604:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176384.504606:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176384.504612:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176384.504630:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176384.504634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176384.504637:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176384.504639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176384.504642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176385.528731:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176385.528743:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176385.528745:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176385.528754:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176385.528759:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176385.528765:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176385.528769:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176385.528773:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176385.528777:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176386.104753:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176386.104758:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176386.104761:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176386.552589:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176386.552619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176386.552621:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176386.552627:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176386.552631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176386.552635:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176386.552638:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176386.552641:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176386.552644:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176387.576577:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176387.576602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176387.576604:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176387.576609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176387.576613:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176387.576616:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176387.576619:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176387.576623:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176387.576626:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176387.577240:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176387.577246:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786176387.577880:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176387.577882:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786176387.578078:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176387.578080:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786176387.578269:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176387.578271:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1786176387.578491:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176387.578493:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000100:00000001:2.0:1786176388.600602:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176388.600610:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176388.600613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176388.600619:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176388.600622:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176388.600626:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176388.600629:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176388.600631:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176388.600634:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176389.176731:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176389.176737:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176389.176741:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176389.624631:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176389.624639:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176389.624642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176389.624649:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176389.624653:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176389.624656:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176389.624660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176389.624663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176389.624666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176390.648635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176390.648643:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176390.648646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176390.648650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176390.648654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176390.648657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176390.648660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176390.648663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176390.648666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176391.672632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176391.672638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176391.672640:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176391.672644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176391.672647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176391.672650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176391.672653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176391.672655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176391.672656:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176392.248667:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176392.248671:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176392.248673:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176392.696620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176392.696629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176392.696632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176392.696638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176392.696642:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176392.696646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176392.696649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176392.696652:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176392.696655:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176392.697148:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176392.697154:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786176392.697797:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176392.697799:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786176392.698016:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176392.698018:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786176392.698243:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176392.698245:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1786176392.698490:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176392.698492:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000100:00000001:2.0:1786176393.720645:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176393.720653:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176393.720656:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176393.720661:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176393.720666:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176393.720670:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176393.720672:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176393.720674:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176393.720677:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176394.744579:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176394.744589:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176394.744591:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176394.744598:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176394.744602:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176394.744606:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176394.744610:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176394.744612:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176394.744615:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176395.320740:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176395.320745:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176395.320748:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176395.768577:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176395.768583:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176395.768585:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176395.768589:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176395.768599:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176395.768601:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176395.768604:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176395.768606:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176395.768608:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176396.792567:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176396.792572:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176396.792574:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176396.792578:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176396.792581:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176396.792585:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176396.792588:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176396.792590:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176396.792592:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176397.816622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176397.816628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176397.816631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176397.816636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176397.816640:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176397.816644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176397.816647:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176397.816650:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176397.816654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176397.817278:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176397.817287:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176397.817893:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176397.817895:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176397.818179:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176397.818182:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176397.818467:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176397.818469:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176397.818803:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176397.818805:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000020:00000001:2.0:1786176398.392675:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176398.392679:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176398.392682:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176398.840592:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176398.840601:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176398.840603:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176398.840608:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176398.840611:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176398.840613:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176398.840621:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176398.840624:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176398.840626:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176399.864657:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176399.864665:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176399.864668:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176399.864674:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176399.864687:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176399.864692:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176399.864697:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176399.864700:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176399.864703:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176400.888589:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176400.888597:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176400.888598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176400.888603:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176400.888607:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176400.888610:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176400.888612:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176400.888615:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176400.888617:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176401.464687:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176401.464690:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176401.464693:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176401.912594:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176401.912602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176401.912604:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176401.912610:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176401.912615:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176401.912619:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176401.912622:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176401.912625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176401.912628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176402.936582:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176402.936589:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176402.936591:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176402.936596:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176402.936600:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176402.936602:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176402.936605:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176402.936608:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176402.936609:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176402.937121:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176402.937474:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176402.937940:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176402.938170:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176402.938311:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176403.960643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176403.960650:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176403.960652:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176403.960655:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176403.960658:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176403.960661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176403.960663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176403.960664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176403.960666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176404.536793:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176404.536799:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176404.536802:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176404.984704:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176404.984711:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176404.984713:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176404.984719:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176404.984722:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176404.984726:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176404.984729:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176404.984732:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176404.984735:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176406.008803:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176406.008810:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176406.008811:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176406.008816:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176406.008818:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176406.008821:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176406.008824:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176406.008826:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176406.008828:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176407.032656:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176407.032663:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176407.032664:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176407.032669:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176407.032672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176407.032675:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176407.032677:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176407.032679:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176407.032681:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176407.608720:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176407.608724:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176407.608726:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176408.056661:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176408.056666:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176408.056668:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176408.056672:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176408.056674:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176408.056678:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176408.056681:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176408.056684:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176408.056687:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176408.057286:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176408.057292:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176408.058095:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176408.058100:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176408.058488:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176408.058491:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176408.058925:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176408.058928:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176408.059245:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176408.059247:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1786176409.080699:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176409.080705:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176409.080707:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176409.080711:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176409.080714:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176409.080717:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176409.080719:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176409.080721:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176409.080723:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176410.104608:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176410.104616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176410.104619:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176410.104625:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176410.104629:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176410.104633:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176410.104636:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176410.104639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176410.104642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176410.680714:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176410.680717:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176410.680720:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176411.128614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176411.128621:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176411.128623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176411.128626:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176411.128629:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176411.128632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176411.128635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176411.128637:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176411.128639:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176412.152592:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176412.152601:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176412.152604:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176412.152609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176412.152613:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176412.152617:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176412.152621:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176412.152624:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176412.152628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176413.176650:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176413.176656:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176413.176659:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176413.176665:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176413.176670:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176413.176673:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176413.176677:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176413.176680:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176413.176684:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176413.177134:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176413.177138:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176413.177838:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176413.177840:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176413.178031:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176413.178032:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176413.178210:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176413.178212:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176413.178483:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176413.178485:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:2.0:1786176413.752662:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176413.752668:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176413.752672:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176414.200591:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176414.200600:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176414.200602:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176414.200607:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176414.200611:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176414.200615:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176414.200618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176414.200621:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176414.200623:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786176415.096616:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176415.096625:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176415.096631:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:2.0:1786176415.224639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176415.224646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176415.224648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176415.224654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176415.224668:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176415.224673:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176415.224676:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176415.224679:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176415.224683:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176416.248635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176416.248645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176416.248648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176416.248656:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176416.248660:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176416.248666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176416.248670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176416.248674:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176416.248679:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176416.824738:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176416.824746:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176416.824751:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176417.272640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176417.272649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176417.272652:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176417.272659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176417.272664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176417.272668:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176417.272671:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176417.272674:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176417.272677:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176418.296597:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176418.296607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176418.296610:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176418.296617:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176418.296621:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176418.296625:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176418.296627:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176418.296629:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176418.296631:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176418.297440:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176418.297456:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786176418.298371:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176418.298374:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786176418.298737:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176418.298740:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786176418.299076:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176418.299078:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786176418.299411:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176418.299415:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1786176419.320614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176419.320626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176419.320628:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176419.320635:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176419.320639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176419.320642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176419.320645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176419.320648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176419.320651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176419.896786:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176419.896793:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176419.896799:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176420.344599:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176420.344608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176420.344610:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176420.344615:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176420.344619:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176420.344623:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176420.344626:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176420.344629:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176420.344631:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176421.368631:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176421.368637:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176421.368639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176421.368644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176421.368646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176421.368649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176421.368651:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176421.368653:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176421.368655:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176422.392624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176422.392632:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176422.392635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176422.392641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176422.392646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176422.392649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176422.392653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176422.392656:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176422.392659:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176422.968645:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176422.968648:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176422.968651:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176423.416581:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176423.416587:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176423.416589:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176423.416594:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176423.416598:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176423.416601:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176423.416604:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176423.416607:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176423.416610:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176423.417131:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176423.417135:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786176423.418114:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176423.418117:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786176423.418563:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176423.418565:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786176423.418803:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176423.418805:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786176423.419262:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176423.419275:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1786176424.440691:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176424.440706:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176424.440726:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176424.440734:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176424.440740:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176424.440746:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176424.440751:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176424.440768:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176424.440774:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176425.464624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176425.464634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176425.464637:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176425.464643:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176425.464648:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176425.464653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176425.464657:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176425.464661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176425.464664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176426.040700:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176426.040705:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176426.040708:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176426.488639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176426.488647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176426.488650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176426.488668:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176426.488672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176426.488676:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176426.488680:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176426.488682:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176426.488685:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176427.512590:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176427.512597:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176427.512600:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176427.512606:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176427.512610:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176427.512614:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176427.512618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176427.512622:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176427.512626:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176428.536647:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176428.536657:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176428.536660:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176428.536669:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176428.536674:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176428.536679:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176428.536683:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176428.536687:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176428.536691:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176428.537848:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176428.537863:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786176428.538903:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176428.538907:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786176428.539282:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176428.539285:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786176428.539662:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176428.539665:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786176428.540001:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176428.540004:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:2.0:1786176429.112826:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176429.112835:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176429.112842:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176429.560613:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176429.560622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176429.560625:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176429.560632:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176429.560637:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176429.560641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176429.560645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176429.560649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176429.560652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176430.584629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176430.584638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176430.584640:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176430.584648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176430.584652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176430.584657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176430.584661:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176430.584664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176430.584667:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176431.608591:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176431.608600:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176431.608602:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176431.608608:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176431.608612:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176431.608616:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176431.608629:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176431.608633:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176431.608637:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176432.184739:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176432.184775:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176432.184780:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176432.632650:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176432.632659:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176432.632663:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176432.632704:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176432.632709:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176432.632713:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176432.632716:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176432.632720:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176432.632723:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176433.656582:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176433.656594:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176433.656596:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176433.656601:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176433.656604:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176433.656607:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176433.656610:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176433.656616:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176433.656619:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176433.657246:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176433.657252:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786176433.657933:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176433.657937:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786176433.658282:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176433.658284:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786176433.658580:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176433.658584:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1786176433.658988:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176433.658990:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1786176434.680604:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176434.680613:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176434.680616:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176434.680622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176434.680626:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176434.680630:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176434.680633:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176434.680636:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176434.680638:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176435.256659:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176435.256663:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176435.256665:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176435.704583:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176435.704589:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176435.704591:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176435.704595:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176435.704598:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176435.704601:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176435.704603:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176435.704605:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176435.704607:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176436.728633:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176436.728641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176436.728644:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176436.728650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176436.728655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176436.728659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176436.728670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176436.728673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176436.728675:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176437.752564:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176437.752573:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176437.752576:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176437.752582:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176437.752586:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176437.752590:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176437.752594:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176437.752596:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176437.752599:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176438.328706:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176438.328711:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176438.328715:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176438.776590:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176438.776596:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176438.776598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176438.776603:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176438.776606:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176438.776610:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176438.776613:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176438.776615:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176438.776618:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176438.777149:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176438.777153:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176438.777734:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176438.777736:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176438.777955:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176438.777957:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176438.778159:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176438.778160:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176438.778375:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176438.778377:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:2.0:1786176439.800598:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176439.800605:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176439.800608:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176439.800613:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176439.800616:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176439.800620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176439.800622:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176439.800625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176439.800628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176440.824597:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176440.824604:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176440.824606:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176440.824611:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176440.824614:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176440.824617:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176440.824620:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176440.824622:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176440.824624:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176441.400709:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176441.400716:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176441.400720:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176441.848628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176441.848634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176441.848635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176441.848640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176441.848643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176441.848646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176441.848648:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176441.848651:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176441.848653:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176442.872586:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176442.872592:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176442.872594:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176442.872598:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176442.872601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176442.872603:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176442.872605:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176442.872607:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176442.872609:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176443.896615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176443.896622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176443.896624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176443.896629:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176443.896633:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176443.896638:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176443.896642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176443.896647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176443.896652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176443.897057:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176443.897061:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176443.897474:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176443.897476:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176443.897681:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176443.897682:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176443.897875:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176443.897876:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176443.898065:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176443.898067:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:2.0:1786176444.472678:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176444.472682:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176444.472685:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176444.920568:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176444.920574:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176444.920576:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176444.920580:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176444.920583:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176444.920586:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176444.920588:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176444.920590:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176444.920593:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176445.944578:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176445.944585:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176445.944586:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176445.944592:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176445.944596:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176445.944600:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176445.944603:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176445.944607:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176445.944610:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176446.968605:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176446.968612:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176446.968613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176446.968618:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176446.968621:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176446.968623:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176446.968626:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176446.968628:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176446.968630:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176447.544746:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176447.544754:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176447.544759:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176447.992628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176447.992639:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176447.992642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176447.992650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176447.992655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176447.992661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176447.992664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176447.992668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176447.992672:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176449.016611:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176449.016619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176449.016626:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176449.016634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176449.016641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176449.016647:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176449.016655:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176449.016661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176449.016666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176449.017476:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176449.017488:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176449.018487:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176449.018492:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176449.018877:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176449.018880:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176449.019262:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176449.019264:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176449.019622:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176449.019625:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1786176450.040625:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176450.040632:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176450.040635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176450.040640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176450.040644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176450.040649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176450.040652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176450.040655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176450.040658:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176450.616663:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176450.616667:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176450.616681:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176451.064563:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176451.064572:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176451.064575:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176451.064581:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176451.064585:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176451.064590:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176451.064593:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176451.064597:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176451.064600:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176452.088658:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176452.088664:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176452.088666:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176452.088682:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176452.088702:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176452.088705:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176452.088707:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176452.088709:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176452.088711:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786176452.984639:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176452.984650:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176452.984657:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:2.0:1786176453.112618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176453.112627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176453.112630:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176453.112637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176453.112642:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176453.112646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176453.112650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176453.112654:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176453.112657:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176453.688706:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176453.688710:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176453.688712:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176454.136639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176454.136647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176454.136651:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176454.136658:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176454.136663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176454.136668:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176454.136674:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176454.136680:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176454.136686:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176454.137204:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176454.137213:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176454.137838:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176454.137839:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176454.138111:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176454.138113:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176454.138369:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176454.138371:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176454.138644:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176454.138646:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1786176455.160573:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176455.160579:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176455.160581:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176455.160585:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176455.160589:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176455.160591:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176455.160594:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176455.160596:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176455.160598:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176456.184594:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176456.184602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176456.184605:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176456.184610:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176456.184614:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176456.184619:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176456.184622:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176456.184625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176456.184627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176456.760638:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176456.760642:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176456.760645:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176457.208596:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176457.208602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176457.208604:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176457.208609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176457.208612:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176457.208615:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176457.208617:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176457.208625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176457.208628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176458.232641:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176458.232648:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176458.232650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176458.232654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176458.232657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176458.232661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176458.232663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176458.232665:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176458.232668:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176459.256585:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176459.256591:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176459.256594:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176459.256598:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176459.256601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176459.256604:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176459.256607:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176459.256609:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176459.256612:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176459.257281:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176459.257289:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176459.257852:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176459.257855:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176459.258182:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176459.258185:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176459.258414:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176459.258416:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176459.258619:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176459.258621:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:2.0:1786176459.832714:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176459.832718:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176459.832721:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176460.280623:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176460.280629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176460.280630:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176460.280635:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176460.280638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176460.280641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176460.280644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176460.280646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176460.280648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176461.304583:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176461.304592:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176461.304595:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176461.304601:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176461.304605:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176461.304608:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176461.304612:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176461.304615:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176461.304618:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176462.328649:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176462.328662:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176462.328665:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176462.328675:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176462.328695:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176462.328702:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176462.328706:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176462.328713:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176462.328717:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176462.904746:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176462.904752:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176462.904756:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176463.352706:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176463.352715:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176463.352717:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176463.352723:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176463.352726:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176463.352730:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176463.352743:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176463.352746:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176463.352748:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176464.376644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176464.376655:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176464.376661:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176464.376676:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176464.376690:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176464.376696:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176464.376704:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176464.376711:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176464.376718:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176464.377272:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176464.377280:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176464.377927:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176464.377930:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176464.378241:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176464.378244:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176464.378579:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176464.378581:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176464.378907:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176464.378910:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1786176465.400616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176465.400622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176465.400624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176465.400628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176465.400631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176465.400634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176465.400637:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176465.400639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176465.400641:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176465.976667:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176465.976671:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176465.976673:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176466.424590:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176466.424596:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176466.424597:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176466.424602:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176466.424605:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176466.424607:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176466.424610:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176466.424612:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176466.424614:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176467.448616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176467.448622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176467.448624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176467.448629:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176467.448632:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176467.448635:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176467.448638:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176467.448641:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176467.448644:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176468.472590:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176468.472596:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176468.472598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176468.472603:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176468.472605:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176468.472608:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176468.472610:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176468.472612:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176468.472614:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176469.048850:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176469.048855:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176469.048858:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176469.496685:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176469.496694:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176469.496697:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176469.496709:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176469.496719:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176469.496727:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176469.496738:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176469.496744:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176469.496750:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176469.497311:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176469.497782:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176469.498160:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176469.498341:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176469.498538:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176470.520644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176470.520650:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176470.520652:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176470.520656:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176470.520659:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176470.520661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176470.520664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176470.520666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176470.520667:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176471.544677:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176471.544683:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176471.544685:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176471.544689:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176471.544698:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176471.544701:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176471.544704:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176471.544706:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176471.544708:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176472.120730:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176472.120736:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176472.120740:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176472.568708:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176472.568715:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176472.568716:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176472.568721:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176472.568723:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176472.568726:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176472.568728:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176472.568730:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176472.568732:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176473.592683:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176473.592689:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176473.592691:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176473.592695:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176473.592698:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176473.592702:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176473.592704:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176473.592706:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176473.592708:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176474.616639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176474.616646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176474.616649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176474.616654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176474.616660:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176474.616663:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176474.616667:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176474.616670:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176474.616674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176474.617655:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176474.617659:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176474.618045:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176474.618048:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176474.618351:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176474.618354:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176474.618678:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176474.618680:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176474.619059:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176474.619061:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:2.0:1786176475.192710:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176475.192714:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176475.192716:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176475.640667:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176475.640673:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176475.640675:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176475.640679:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176475.640683:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176475.640686:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176475.640688:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176475.640690:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176475.640692:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176476.664574:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176476.664580:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176476.664581:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176476.664586:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176476.664589:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176476.664592:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176476.664594:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176476.664596:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176476.664598:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176477.688624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176477.688631:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176477.688634:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176477.688640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176477.688645:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176477.688648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176477.688651:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176477.688655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176477.688659:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176478.264645:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176478.264649:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176478.264652:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176478.712574:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176478.712580:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176478.712582:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176478.712586:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176478.712589:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176478.712592:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176478.712594:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176478.712596:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176478.712599:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176479.736609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176479.736615:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176479.736618:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176479.736622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176479.736626:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176479.736631:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176479.736634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176479.736638:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176479.736642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176479.737157:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176479.737164:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176479.737785:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176479.737787:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176479.738046:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176479.738048:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176479.738261:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176479.738263:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176479.738443:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176479.738446:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1786176480.760571:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176480.760578:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176480.760579:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176480.760584:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176480.760587:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176480.760590:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176480.760592:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176480.760594:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176480.760596:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176481.336726:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176481.336730:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176481.336733:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176481.784594:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176481.784603:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176481.784605:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176481.784611:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176481.784615:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176481.784619:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176481.784622:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176481.784625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176481.784627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176482.808596:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176482.808603:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176482.808605:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176482.808609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176482.808613:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176482.808616:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176482.808618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176482.808620:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176482.808622:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176483.832596:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176483.832603:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176483.832604:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176483.832609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176483.832612:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176483.832615:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176483.832617:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176483.832619:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176483.832621:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176484.408874:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176484.408882:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176484.408887:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176484.856658:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176484.856665:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176484.856668:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176484.856674:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176484.856678:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176484.856681:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176484.856683:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176484.856686:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176484.856688:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176484.857219:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176484.857224:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176484.857891:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176484.857893:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176484.858094:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176484.858096:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176484.858282:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176484.858284:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176484.858464:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176484.858466:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1786176485.880710:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176485.880718:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176485.880720:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176485.880726:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176485.880730:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176485.880734:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176485.880737:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176485.880739:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176485.880742:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176486.904682:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176486.904693:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176486.904697:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176486.904705:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176486.904709:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176486.904714:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176486.904717:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176486.904721:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176486.904725:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176487.480712:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176487.480716:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176487.480719:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176487.928588:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176487.928597:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176487.928598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176487.928604:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176487.928607:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176487.928611:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176487.928625:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176487.928629:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176487.928632:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176488.952630:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176488.952639:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176488.952641:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176488.952647:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176488.952651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176488.952655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176488.952657:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176488.952660:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176488.952662:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176489.976629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176489.976653:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176489.976655:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176489.976661:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176489.976664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176489.976667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176489.976670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176489.976672:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176489.976674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176489.977622:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176489.977627:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176489.978546:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176489.978548:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176489.978815:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176489.978817:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176489.979075:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176489.979077:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176489.979369:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176489.979371:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:2.0:1786176490.552711:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176490.552715:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176490.552717:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176491.000674:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176491.000682:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176491.000685:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176491.000692:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176491.000696:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176491.000699:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176491.000701:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176491.000704:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176491.000705:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786176491.896675:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176491.896683:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176491.896687:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:2.0:1786176492.024572:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176492.024603:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176492.024605:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176492.024610:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176492.024613:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176492.024616:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176492.024618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176492.024621:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176492.024623:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176493.048648:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176493.048657:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176493.048659:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176493.048665:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176493.048668:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176493.048672:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176493.048675:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176493.048679:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176493.048682:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176493.624788:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176493.624794:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176493.624798:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176494.072660:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176494.072672:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176494.072675:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176494.072683:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176494.072686:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176494.072690:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176494.072693:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176494.072695:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176494.072698:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176495.096665:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176495.096671:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176495.096675:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176495.096680:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176495.096685:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176495.096690:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176495.096695:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176495.096699:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176495.096705:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176495.097157:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176495.097163:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176495.097973:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176495.097975:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176495.098161:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176495.098163:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176495.098402:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176495.098404:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176495.098756:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176495.098758:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:2.0:1786176496.120686:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176496.120695:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176496.120698:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176496.120706:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176496.120712:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176496.120717:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176496.120721:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176496.120725:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176496.120729:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176496.696808:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176496.696818:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176496.696824:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176497.144676:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176497.144685:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176497.144688:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176497.144707:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176497.144712:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176497.144717:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176497.144721:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176497.144724:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176497.144728:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176498.168574:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176498.168580:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176498.168582:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176498.168587:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176498.168589:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176498.168592:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176498.168594:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176498.168596:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176498.168598:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176499.192609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176499.192615:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176499.192617:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176499.192622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176499.192625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176499.192628:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176499.192630:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176499.192633:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176499.192635:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176499.768859:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176499.768865:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176499.768870:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176500.216647:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176500.216652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176500.216658:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176500.216664:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176500.216670:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176500.216675:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176500.216681:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176500.216686:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176500.216690:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176500.217492:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176500.217565:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786176500.218351:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176500.218353:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786176500.218593:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176500.218595:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786176500.218782:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176500.218784:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786176500.218993:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176500.218995:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:2.0:1786176501.240601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176501.240608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176501.240611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176501.240616:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176501.240620:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176501.240624:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176501.240628:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176501.240630:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176501.240633:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176502.264624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176502.264631:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176502.264633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176502.264638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176502.264641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176502.264643:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176502.264646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176502.264648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176502.264650:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176502.840694:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176502.840698:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176502.840702:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176503.288626:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176503.288633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176503.288635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176503.288639:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176503.288642:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176503.288645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176503.288647:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176503.288649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176503.288651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176504.312639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176504.312646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176504.312648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176504.312652:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176504.312655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176504.312658:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176504.312661:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176504.312663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176504.312665:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176505.336606:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176505.336614:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176505.336627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176505.336637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176505.336643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176505.336647:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176505.336652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176505.336655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176505.336659:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176505.337177:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176505.337182:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786176505.337865:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176505.337870:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786176505.338121:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176505.338123:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786176505.338311:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176505.338313:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786176505.338702:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176505.338703:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000020:00000001:2.0:1786176505.912708:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176505.912713:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176505.912717:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176506.360634:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176506.360641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176506.360642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176506.360647:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176506.360650:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176506.360653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176506.360655:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176506.360657:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176506.360659:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176507.384585:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176507.384593:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176507.384595:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176507.384601:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176507.384606:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176507.384609:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176507.384613:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176507.384616:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176507.384619:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176508.408615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176508.408622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176508.408623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176508.408628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176508.408631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176508.408634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176508.408636:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176508.408638:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176508.408641:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176508.984718:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176508.984722:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176508.984725:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176509.432570:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176509.432578:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176509.432580:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176509.432586:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176509.432590:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176509.432594:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176509.432597:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176509.432600:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176509.432603:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176510.456620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176510.456626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176510.456628:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176510.456633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176510.456638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176510.456642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176510.456646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176510.456649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176510.456653:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176510.457233:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176510.457238:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1786176510.457815:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176510.457817:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1786176510.458039:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176510.458041:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1786176510.458222:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176510.458223:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1786176510.458412:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176510.458414:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000100:00000001:2.0:1786176511.480609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176511.480616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176511.480618:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176511.480622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176511.480625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176511.480628:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176511.480630:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176511.480632:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176511.480634:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176512.056776:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176512.056784:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176512.056790:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176512.504626:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176512.504631:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176512.504633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176512.504638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176512.504641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176512.504644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176512.504646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176512.504648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176512.504650:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176513.528642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176513.528651:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176513.528653:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176513.528659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176513.528663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176513.528667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176513.528670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176513.528673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176513.528676:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176514.552633:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176514.552642:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176514.552645:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176514.552651:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176514.552655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176514.552659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176514.552663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176514.552666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176514.552669:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176515.128720:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176515.128724:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176515.128728:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176515.576593:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176515.576600:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176515.576604:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176515.576609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176515.576613:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176515.576617:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176515.576621:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176515.576625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176515.576628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176515.577127:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176515.577132:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176515.577933:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176515.577936:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176515.578243:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176515.578245:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176515.578587:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176515.578589:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176515.578889:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176515.578892:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1786176516.600604:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176516.600611:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176516.600613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176516.600617:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176516.600620:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176516.600623:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176516.600626:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176516.600628:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176516.600630:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176517.624586:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176517.624592:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176517.624594:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176517.624598:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176517.624601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176517.624604:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176517.624606:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176517.624609:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176517.624611:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176518.200688:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176518.200691:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176518.200693:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176518.648609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176518.648616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176518.648618:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176518.648623:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176518.648625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176518.648629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176518.648631:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176518.648633:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176518.648635:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176519.672639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176519.672647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176519.672649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176519.672656:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176519.672660:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176519.672664:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176519.672667:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176519.672670:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176519.672673:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176520.696607:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176520.696622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176520.696626:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176520.696633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176520.696637:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176520.696640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176520.696644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176520.696649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176520.696654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176520.697134:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176520.697139:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176520.697844:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176520.697845:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176520.698072:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176520.698074:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176520.698265:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176520.698267:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176520.698562:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176520.698565:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:2.0:1786176521.272665:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176521.272668:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176521.272670:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176521.720618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176521.720628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176521.720630:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176521.720637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176521.720641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176521.720645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176521.720649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176521.720653:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176521.720656:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176522.744627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176522.744635:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176522.744637:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176522.744644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176522.744648:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176522.744652:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176522.744656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176522.744659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176522.744662:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176523.768628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176523.768636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176523.768638:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176523.768644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176523.768648:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176523.768652:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176523.768656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176523.768659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176523.768662:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176524.344685:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176524.344689:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176524.344693:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176524.792634:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176524.792642:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176524.792644:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176524.792649:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176524.792652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176524.792655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176524.792658:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176524.792660:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176524.792662:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176525.816650:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176525.816660:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176525.816664:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176525.816672:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176525.816677:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176525.816684:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176525.816690:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176525.816696:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176525.816702:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176525.817263:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176525.817268:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176525.818083:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176525.818085:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176525.818321:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176525.818323:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176525.818624:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176525.818625:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176525.818935:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176525.818937:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:2.0:1786176526.840637:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176526.840646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176526.840648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176526.840665:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176526.840669:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176526.840674:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176526.840677:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176526.840680:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176526.840683:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176527.416748:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176527.416752:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176527.416756:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176527.864643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176527.864653:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176527.864655:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176527.864662:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176527.864666:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176527.864670:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176527.864674:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176527.864677:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176527.864680:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176528.888608:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176528.888621:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176528.888623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176528.888631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176528.888636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176528.888640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176528.888644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176528.888648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176528.888652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786176529.784632:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176529.784643:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176529.784649:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:2.0:1786176529.912626:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176529.912636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176529.912638:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176529.912644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176529.912648:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176529.912652:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176529.912655:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176529.912658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176529.912661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176530.488758:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176530.488762:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176530.488765:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176530.936630:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176530.936638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176530.936640:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176530.936657:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176530.936662:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176530.936666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176530.936670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176530.936673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176530.936677:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176530.937340:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176530.937346:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176530.938004:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176530.938007:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176530.938347:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176530.938350:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176530.938699:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176530.938701:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176530.939015:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176530.939018:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1786176531.960569:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176531.960576:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176531.960578:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176531.960582:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176531.960585:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176531.960588:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176531.960590:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176531.960592:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176531.960594:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176532.984629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176532.984637:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176532.984639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176532.984645:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176532.984650:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176532.984654:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176532.984657:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176532.984661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176532.984664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176533.560683:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176533.560687:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176533.560691:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176534.008639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176534.008647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176534.008650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176534.008656:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176534.008660:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176534.008664:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176534.008666:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176534.008669:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176534.008672:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176535.032614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176535.032620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176535.032622:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176535.032625:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176535.032628:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176535.032632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176535.032634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176535.032636:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176535.032638:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176536.056635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176536.056642:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176536.056646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176536.056652:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176536.056658:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176536.056662:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176536.056668:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176536.056671:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176536.056674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176536.057203:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176536.057672:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176536.057860:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176536.058035:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176536.058216:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786176536.632721:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176536.632725:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176536.632728:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176537.080586:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176537.080594:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176537.080596:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176537.080602:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176537.080606:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176537.080610:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176537.080613:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176537.080616:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176537.080619:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176538.104615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176538.104622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176538.104624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176538.104629:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176538.104632:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176538.104635:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176538.104638:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176538.104640:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176538.104642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176539.128589:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176539.128597:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176539.128600:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176539.128605:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176539.128610:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176539.128614:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176539.128617:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176539.128620:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176539.128624:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176539.704703:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176539.704707:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176539.704710:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176540.152640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176540.152647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176540.152649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176540.152653:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176540.152657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176540.152659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176540.152662:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176540.152664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176540.152667:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176541.176614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176541.176620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176541.176624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176541.176630:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176541.176635:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176541.176640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176541.176644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176541.176646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176541.176650:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176541.177283:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176541.177291:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176541.177863:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176541.177867:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176541.178395:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176541.178398:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176541.178856:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176541.178858:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176541.179164:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176541.179167:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1786176542.200625:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176542.200631:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176542.200633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176542.200638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176542.200641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176542.200644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176542.200647:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176542.200649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176542.200651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176542.776664:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176542.776668:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176542.776671:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176543.224625:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176543.224632:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176543.224634:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176543.224638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176543.224642:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176543.224645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176543.224647:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176543.224649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176543.224651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176544.248613:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176544.248620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176544.248622:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176544.248627:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176544.248630:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176544.248633:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176544.248635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176544.248638:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176544.248640:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176545.272630:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176545.272637:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176545.272640:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176545.272645:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176545.272649:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176545.272653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176545.272656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176545.272659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176545.272661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176545.848772:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176545.848778:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176545.848781:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176546.296618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176546.296625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176546.296627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176546.296633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176546.296638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176546.296641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176546.296645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176546.296648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176546.296655:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176546.297216:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176546.297222:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176546.297887:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176546.297891:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176546.298352:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176546.298354:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176546.298661:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176546.298663:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176546.298855:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176546.298857:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1786176547.320654:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176547.320661:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176547.320663:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176547.320668:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176547.320673:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176547.320676:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176547.320678:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176547.320680:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176547.320684:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176548.344565:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176548.344572:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176548.344574:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176548.344579:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176548.344582:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176548.344585:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176548.344587:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176548.344589:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176548.344591:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176548.920717:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176548.920720:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176548.920723:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176549.368633:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176549.368641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176549.368643:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176549.368648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176549.368652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176549.368656:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176549.368659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176549.368661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176549.368664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176550.392643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176550.392652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176550.392654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176550.392660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176550.392664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176550.392668:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176550.392670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176550.392673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176550.392676:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176551.416616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176551.416623:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176551.416625:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176551.416633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176551.416637:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176551.416641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176551.416645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176551.416649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176551.416653:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176551.417212:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176551.417217:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176551.417928:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176551.417931:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176551.418211:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176551.418212:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176551.418432:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176551.418433:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176551.418713:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176551.418715:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:2.0:1786176551.992716:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176551.992720:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176551.992723:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176552.440677:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176552.440683:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176552.440685:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176552.440689:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176552.440692:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176552.440695:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176552.440698:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176552.440700:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176552.440701:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176553.464614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176553.464620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176553.464622:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176553.464626:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176553.464629:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176553.464632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176553.464634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176553.464636:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176553.464639:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176554.488583:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176554.488589:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176554.488590:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176554.488595:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176554.488597:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176554.488601:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176554.488603:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176554.488605:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176554.488607:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176555.064715:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176555.064718:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176555.064722:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176555.512640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176555.512648:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176555.512650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176555.512654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176555.512658:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176555.512660:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176555.512663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176555.512665:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176555.512668:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176556.536595:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176556.536601:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176556.536603:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176556.536608:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176556.536611:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176556.536614:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176556.536617:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176556.536619:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176556.536622:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176556.537204:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176556.537209:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176556.537886:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176556.537888:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176556.538108:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176556.538110:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176556.538295:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176556.538297:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176556.538560:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176556.538562:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1786176557.560599:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176557.560606:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176557.560607:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176557.560611:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176557.560614:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176557.560617:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176557.560618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176557.560621:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176557.560623:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176558.136685:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176558.136688:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176558.136691:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176558.584603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176558.584610:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176558.584611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176558.584616:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176558.584619:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176558.584621:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176558.584624:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176558.584626:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176558.584628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176559.608631:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176559.608639:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176559.608641:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176559.608646:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176559.608650:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176559.608653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176559.608656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176559.608659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176559.608661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176560.632673:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176560.632683:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176560.632685:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176560.632691:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176560.632695:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176560.632699:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176560.632701:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176560.632704:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176560.632706:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176561.208721:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176561.208728:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176561.208750:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176561.656640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176561.656649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176561.656653:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176561.656684:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176561.656698:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176561.656708:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176561.656722:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176561.656731:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176561.656737:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176561.657351:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176561.657357:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1786176561.658069:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176561.658072:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1786176561.658360:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176561.658363:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1786176561.658714:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176561.658717:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1786176561.659077:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176561.659080:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:2.0:1786176562.680667:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176562.680677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176562.680679:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176562.680685:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176562.680689:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176562.680693:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176562.680696:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176562.680699:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176562.680702:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176563.704608:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176563.704620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176563.704623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176563.704641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176563.704646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176563.704651:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176563.704655:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176563.704658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176563.704661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176564.280749:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176564.280753:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176564.280756:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176564.728630:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176564.728636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176564.728638:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176564.728643:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176564.728646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176564.728649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176564.728652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176564.728654:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176564.728656:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176565.752577:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176565.752585:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176565.752588:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176565.752594:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176565.752598:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176565.752602:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176565.752605:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176565.752608:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176565.752610:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176566.776639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176566.776647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176566.776652:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176566.776659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176566.776665:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176566.776671:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176566.776675:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176566.776678:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176566.776683:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176566.777194:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176566.777199:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176566.777841:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176566.777843:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176566.778084:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176566.778085:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176566.778267:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176566.778269:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176566.778450:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176566.778453:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:2.0:1786176567.352647:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176567.352652:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176567.352656:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176567.800566:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176567.800574:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176567.800577:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176567.800583:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176567.800587:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176567.800591:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176567.800605:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176567.800608:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176567.800612:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786176568.696630:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176568.696640:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176568.696646:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:2.0:1786176568.824619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176568.824626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176568.824628:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176568.824634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176568.824639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176568.824642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176568.824646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176568.824649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176568.824652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176569.848619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176569.848625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176569.848627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176569.848632:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176569.848635:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176569.848638:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176569.848640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176569.848642:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176569.848644:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176570.424748:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176570.424751:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176570.424754:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176570.872621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176570.872627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176570.872629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176570.872633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176570.872636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176570.872640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176570.872642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176570.872644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176570.872646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176571.896602:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176571.896611:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176571.896614:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176571.896623:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176571.896630:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176571.896636:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176571.896643:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176571.896647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176571.896654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176571.897180:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176571.897186:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176571.897904:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176571.897909:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176571.898188:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176571.898190:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176571.898383:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176571.898385:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176571.898722:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176571.898724:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:2.0:1786176572.920665:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176572.920671:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176572.920673:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176572.920677:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176572.920680:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176572.920684:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176572.920686:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176572.920688:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176572.920690:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176573.496691:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176573.496695:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176573.496697:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176573.944577:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176573.944583:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176573.944585:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176573.944590:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176573.944592:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176573.944596:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176573.944598:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176573.944600:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176573.944602:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176574.968609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176574.968615:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176574.968617:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176574.968622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176574.968624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176574.968628:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176574.968630:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176574.968632:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176574.968634:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176575.992589:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176575.992596:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176575.992598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176575.992602:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176575.992606:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176575.992610:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176575.992613:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176575.992616:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176575.992619:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176576.568727:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176576.568731:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176576.568734:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176577.016648:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176577.016660:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176577.016671:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176577.016680:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176577.016694:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176577.016704:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176577.016710:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176577.016716:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176577.016722:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176577.017340:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176577.017346:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176577.017936:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176577.017938:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176577.018201:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176577.018203:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176577.018441:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176577.018443:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176577.018642:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176577.018644:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1786176578.040585:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176578.040592:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176578.040598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176578.040603:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176578.040606:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176578.040610:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176578.040612:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176578.040615:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176578.040617:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176579.064626:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176579.064634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176579.064636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176579.064640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176579.064643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176579.064646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176579.064648:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176579.064650:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176579.064653:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176579.640716:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176579.640719:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176579.640722:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176580.088649:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176580.088656:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176580.088658:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176580.088663:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176580.088666:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176580.088669:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176580.088671:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176580.088673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176580.088675:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176581.112639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176581.112646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176581.112648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176581.112653:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176581.112656:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176581.112659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176581.112662:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176581.112664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176581.112666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176582.136623:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176582.136631:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176582.136634:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176582.136641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176582.136647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176582.136652:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176582.136658:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176582.136661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176582.136664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176582.137256:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176582.137263:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176582.137959:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176582.137963:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176582.138381:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176582.138382:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176582.138659:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176582.138661:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176582.138876:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176582.138877:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:2.0:1786176582.712709:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176582.712713:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176582.712716:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176583.160640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176583.160648:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176583.160651:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176583.160657:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176583.160661:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176583.160664:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176583.160668:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176583.160671:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176583.160674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176584.184636:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176584.184644:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176584.184646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176584.184652:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176584.184656:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176584.184660:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176584.184664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176584.184666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176584.184670:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176585.208604:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176585.208610:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176585.208613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176585.208618:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176585.208630:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176585.208634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176585.208640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176585.208644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176585.208646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176585.784729:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176585.784733:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176585.784738:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176586.232647:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176586.232655:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176586.232658:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176586.232664:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176586.232702:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176586.232706:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176586.232709:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176586.232712:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176586.232722:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176587.257190:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176587.257196:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786176587.257902:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176587.257904:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786176587.258115:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176587.258118:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786176587.258335:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176587.258337:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786176587.258641:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176587.258643:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:2.0:1786176588.856695:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176588.856699:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176588.856703:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176591.928744:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176591.928747:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176591.928750:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176592.377267:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176592.377274:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786176592.377846:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176592.377849:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786176592.378085:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176592.378086:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786176592.378329:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176592.378331:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786176592.378579:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176592.378581:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:2.0:1786176595.000735:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176595.000741:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176595.000745:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176597.497373:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176597.497384:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176597.498371:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176597.498379:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176597.499055:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176597.499057:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176597.499307:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176597.499309:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176597.499548:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176597.499550:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:2.0:1786176598.072694:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176598.072698:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176598.072702:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176601.144769:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176601.144773:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176601.144776:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176602.616641:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176602.616646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176602.616648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176602.616654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176602.616658:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176602.616662:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176602.616666:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176602.616670:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176602.616673:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176602.617153:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176602.617472:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176602.617892:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176602.618135:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176602.618304:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176603.640711:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176603.640720:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176603.640722:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176603.640727:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176603.640730:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176603.640733:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176603.640736:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176603.640738:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176603.640740:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176604.216669:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176604.216673:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176604.216676:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176604.664637:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176604.664646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176604.664648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176604.664653:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176604.664657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176604.664662:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176604.664665:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176604.664668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176604.664671:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176605.688776:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176605.688783:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176605.688785:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176605.688795:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176605.688801:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176605.688804:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176605.688806:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176605.688809:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176605.688811:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786176606.584635:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786176606.584642:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786176606.584646:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000100:00000001:2.0:1786176606.712682:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176606.712688:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176606.712690:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176606.712694:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176606.712697:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176606.712700:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176606.712702:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176606.712704:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176606.712706:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176607.288750:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176607.288754:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176607.288756:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176607.736692:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176607.736699:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176607.736704:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176607.736711:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176607.736719:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176607.736726:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176607.736731:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176607.736736:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176607.736742:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176607.737305:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176607.737312:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176607.738096:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176607.738099:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176607.738455:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176607.738476:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176607.738833:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176607.738836:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176607.739171:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176607.739174:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1786176608.760725:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176608.760731:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176608.760732:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176608.760736:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176608.760739:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176608.760742:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176608.760745:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176608.760747:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176608.760749:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176609.784627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176609.784633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176609.784636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176609.784640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176609.784644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176609.784646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176609.784649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176609.784651:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176609.784652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176610.360729:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176610.360733:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176610.360736:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176610.808578:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176610.808584:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176610.808586:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176610.808592:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176610.808594:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176610.808598:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176610.808600:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176610.808603:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176610.808605:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176611.832586:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176611.832595:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176611.832597:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176611.832603:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176611.832606:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176611.832611:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176611.832614:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176611.832617:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176611.832620:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176612.856599:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176612.856605:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176612.856609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176612.856616:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176612.856621:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176612.856625:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176612.856629:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176612.856632:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176612.856636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176612.857120:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176612.857126:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176612.857789:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176612.857791:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176612.857978:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176612.857980:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176612.858164:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176612.858166:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176612.858425:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176612.858427:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:2.0:1786176613.432704:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176613.432708:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176613.432711:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176613.880641:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176613.880647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176613.880649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176613.880654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176613.880657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176613.880660:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176613.880663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176613.880665:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176613.880667:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176614.904586:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176614.904595:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176614.904598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176614.904604:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176614.904609:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176614.904613:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176614.904615:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176614.904617:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176614.904619:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176615.928640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176615.928649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176615.928651:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176615.928657:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176615.928661:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176615.928665:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176615.928669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176615.928671:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176615.928674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176616.504669:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176616.504674:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176616.504678:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176616.952643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176616.952651:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176616.952653:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176616.952659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176616.952664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176616.952667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176616.952671:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176616.952674:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176616.952677:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176617.976632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176617.976640:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176617.976642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176617.976649:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176617.976656:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176617.976661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176617.976667:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176617.976671:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176617.976677:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176617.977275:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176617.977281:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786176617.978024:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176617.978029:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786176617.978281:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176617.978283:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786176617.978490:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176617.978492:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786176617.978814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176617.978817:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1786176619.000626:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176619.000635:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176619.000638:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176619.000643:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176619.000646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176619.000650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176619.000652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176619.000654:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176619.000656:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176619.576693:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176619.576697:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176619.576700:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176620.024620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176620.024627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176620.024628:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176620.024633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176620.024636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176620.024639:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176620.024642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176620.024644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176620.024646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176621.048648:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176621.048656:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176621.048658:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176621.048665:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176621.048669:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176621.048672:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176621.048675:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176621.048678:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176621.048680:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176622.072601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176622.072607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176622.072609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176622.072613:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176622.072616:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176622.072619:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176622.072621:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176622.072623:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176622.072625:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176622.648648:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176622.648651:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176622.648654:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176623.096669:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176623.096677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176623.096679:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176623.096685:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176623.096690:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176623.096694:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176623.096699:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176623.096702:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176623.096705:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176623.097404:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176623.097412:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786176623.098043:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176623.098045:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786176623.098289:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176623.098291:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786176623.098622:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176623.098625:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786176623.098837:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176623.098838:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1786176624.120610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176624.120619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176624.120621:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176624.120627:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176624.120631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176624.120635:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176624.120639:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176624.120642:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176624.120646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176625.144621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176625.144630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176625.144632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176625.144639:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176625.144643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176625.144647:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176625.144650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176625.144654:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176625.144657:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176625.720703:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176625.720708:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176625.720711:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176626.168648:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176626.168654:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176626.168655:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176626.168660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176626.168663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176626.168667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176626.168669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176626.168671:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176626.168673:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176627.192632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176627.192639:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176627.192640:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176627.192645:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176627.192648:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176627.192650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176627.192653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176627.192655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176627.192657:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176628.216631:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176628.216643:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176628.216647:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176628.216654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176628.216661:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176628.216676:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176628.216681:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176628.216687:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176628.216693:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176628.217617:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176628.217625:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176628.218301:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176628.218305:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176628.218628:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176628.218630:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176628.218917:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176628.218920:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786176628.219237:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176628.219240:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:2.0:1786176628.792726:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176628.792730:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176628.792732:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176629.240632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176629.240638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176629.240640:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176629.240644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176629.240647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176629.240650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176629.240653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176629.240655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176629.240657:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176630.264629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176630.264635:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176630.264637:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176630.264641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176630.264644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176630.264648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176630.264650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176630.264653:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176630.264654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176631.288627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176631.288634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176631.288636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176631.288640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176631.288650:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176631.288653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176631.288656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176631.288658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176631.288661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176631.864721:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176631.864725:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176631.864728:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176632.312635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176632.312642:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176632.312644:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176632.312649:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176632.312652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176632.312655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176632.312657:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176632.312659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176632.312661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176633.336633:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176633.336641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176633.336644:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176633.336653:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176633.336666:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176633.336670:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176633.336673:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176633.336678:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176633.336681:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176633.337156:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176633.337160:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176633.337816:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176633.337818:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176633.338005:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176633.338007:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176633.338185:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176633.338186:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176633.338451:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176633.338452:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:2.0:1786176634.360608:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176634.360615:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176634.360617:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176634.360621:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176634.360624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176634.360627:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176634.360629:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176634.360632:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176634.360634:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176634.936709:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176634.936714:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176634.936718:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176635.384643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176635.384652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176635.384654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176635.384661:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176635.384664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176635.384670:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176635.384673:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176635.384676:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176635.384679:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176636.408577:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176636.408583:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176636.408585:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176636.408590:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176636.408593:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176636.408596:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176636.408598:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176636.408601:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176636.408604:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176637.432607:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176637.432615:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176637.432618:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176637.432624:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176637.432627:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176637.432631:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176637.432634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176637.432637:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176637.432640:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176638.008733:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176638.008736:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176638.008739:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176638.456620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176638.456628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176638.456633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176638.456640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176638.456646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176638.456650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176638.456655:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176638.456657:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176638.456661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176638.457136:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176638.457140:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786176638.457853:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176638.457858:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786176638.458125:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176638.458127:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786176638.458437:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176638.458439:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786176638.458647:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176638.458648:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000100:00000001:2.0:1786176639.480582:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176639.480587:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176639.480589:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176639.480593:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176639.480596:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176639.480599:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176639.480602:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176639.480604:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176639.480605:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176640.504621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176640.504627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176640.504629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176640.504633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176640.504636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176640.504639:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176640.504641:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176640.504643:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176640.504645:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176641.080684:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176641.080688:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176641.080690:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176641.528585:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176641.528592:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176641.528594:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176641.528598:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176641.528601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176641.528604:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176641.528606:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176641.528609:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176641.528611:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176642.552641:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176642.552647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176642.552649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176642.552654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176642.552657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176642.552660:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176642.552663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176642.552665:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176642.552667:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176643.576562:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176643.576568:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176643.576570:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176643.576575:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176643.576579:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176643.576582:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176643.576585:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176643.576587:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176643.576589:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176643.577282:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176643.577288:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176643.577911:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176643.577913:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176643.578133:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176643.578135:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176643.578369:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176643.578371:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176643.578606:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176643.578608:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:2.0:1786176644.152702:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176644.152706:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176644.152709:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176644.600653:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176644.600662:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176644.600665:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176644.600672:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176644.600677:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176644.600682:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176644.600685:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176644.600688:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176644.600692:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176645.624649:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176645.624655:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176645.624657:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176645.624661:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176645.624664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176645.624668:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176645.624670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176645.624673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176645.624675:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176646.648596:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176646.648602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176646.648604:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176646.648608:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176646.648611:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176646.648614:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176646.648616:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176646.648618:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176646.648621:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176647.224692:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176647.224696:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176647.224700:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176647.672604:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176647.672611:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176647.672613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176647.672618:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176647.672621:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176647.672624:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176647.672626:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176647.672628:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176647.672631:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176648.696635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176648.696641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176648.696643:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176648.696648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176648.696651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176648.696655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176648.696657:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176648.696659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176648.696662:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176648.697465:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176648.697474:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176648.698595:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176648.698597:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176648.698860:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176648.698861:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176648.699039:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176648.699041:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176648.699414:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176648.699417:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1786176649.720590:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176649.720599:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176649.720601:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176649.720607:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176649.720610:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176649.720616:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176649.720618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176649.720621:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176649.720624:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176650.296886:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176650.296892:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176650.296896:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176650.744619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176650.744626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176650.744627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176650.744632:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176650.744634:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176650.744638:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176650.744640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176650.744642:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176650.744644:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176651.768617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176651.768626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176651.768629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176651.768635:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176651.768640:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176651.768644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176651.768647:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176651.768651:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176651.768654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176652.792598:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176652.792604:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176652.792605:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176652.792609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176652.792612:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176652.792615:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176652.792617:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176652.792619:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176652.792621:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176653.368680:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176653.368683:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176653.368686:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176653.817146:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176653.817150:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176653.817807:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176653.817809:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176653.817997:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176653.817998:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176653.818205:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176653.818206:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176653.818442:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176653.818444:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:2.0:1786176656.440758:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176656.440764:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176656.440768:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176658.937319:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176658.937322:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176658.938240:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176658.938249:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176658.939089:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176658.939090:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176658.939275:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176658.939277:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176658.939553:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176658.939555:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:2.0:1786176659.512697:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176659.512700:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176659.512703:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176662.584745:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176662.584753:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176662.584758:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176664.057211:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176664.057217:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176664.057937:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176664.057944:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176664.058396:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176664.058398:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176664.058678:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176664.058680:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176664.058858:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176664.058859:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:2.0:1786176665.656702:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176665.656708:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176665.656712:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176668.728749:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176668.728753:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176668.728757:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176669.177287:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176669.177909:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176669.178562:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176669.178723:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176669.178905:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786176671.800730:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176671.800733:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176671.800736:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:2.0:1786176672.248790:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:2.0:1786176672.248827:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1786176672.248848:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176672.248852:0:2380:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:2.0:1786176672.248856:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176672.248859:0:2380:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1786176672.248864:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176672.248867:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:2.0:1786176672.248871:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176672.248875:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176672.248880:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c43b8
00000400:00000010:2.0:1786176672.248882:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c43b8.
00000100:00000001:2.0:1786176672.248887:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176672.248894:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa7b80 x1872934856282752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176672.248915:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176672.248919:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7b80 x1872934856282752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176694 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176672.248925:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176672.248927:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176672.248936:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176672.248939:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:2.0:1786176672.248942:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176672.248945:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176672.248948:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4d48
00000400:00000010:2.0:1786176672.248950:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4d48.
00000100:00000001:2.0:1786176672.248952:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176672.248957:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa4e00 x1872934856282880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176672.248965:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176672.248968:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4e00 x1872934856282880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176694 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176672.248975:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176672.248976:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176672.248982:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176672.248984:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45a00.
00000400:00000200:2.0:1786176672.248987:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176672.248990:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176672.248993:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4660
00000400:00000010:2.0:1786176672.248995:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4660.
00000100:00000001:2.0:1786176672.248998:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176672.249002:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa5180 x1872934856283008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176672.249010:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176672.249013:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5180 x1872934856283008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176694 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176672.249020:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176672.249022:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176672.249028:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176672.249030:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45200.
00000400:00000200:2.0:1786176672.249033:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176672.249036:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176672.249039:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c46e8
00000400:00000010:2.0:1786176672.249041:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c46e8.
00000100:00000001:2.0:1786176672.249044:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176672.249048:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa4a80 x1872934856283136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176672.249057:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176672.249060:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4a80 x1872934856283136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176694 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176672.249070:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176672.249074:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176672.249082:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176672.249085:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:2.0:1786176672.249090:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176672.249100:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176672.249106:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4440
00000400:00000010:2.0:1786176672.249108:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4440.
00000100:00000001:2.0:1786176672.249111:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176672.249118:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa6d80 x1872934856283264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176672.249132:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176672.249138:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa6d80 x1872934856283264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176694 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176672.249153:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176672.249156:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000200:0.1:1786176674.298005:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176674.298010:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176674.298506:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176674.298553:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176674.299115:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176674.299119:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176674.299507:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176674.299531:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176674.299942:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176674.299944:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:2.0:1786176674.872703:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176674.872708:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176674.872712:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176677.944713:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176677.944717:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176677.944721:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176679.417288:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176679.417292:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176679.418055:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176679.418057:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176679.418267:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176679.418270:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176679.418585:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176679.418588:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176679.418956:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176679.418959:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:2.0:1786176681.016666:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176681.016670:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176681.016674:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176684.088696:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176684.088703:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176684.088707:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176684.537216:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176684.537221:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176684.537863:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176684.537866:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176684.538156:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176684.538158:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176684.538347:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176684.538348:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176684.538568:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176684.538570:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:2.0:1786176687.160682:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176687.160686:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176687.160701:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176689.657332:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176689.657337:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786176689.658096:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176689.658100:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786176689.658347:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176689.658349:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786176689.658605:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176689.658606:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1786176689.658850:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176689.658853:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:2.0:1786176690.232714:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176690.232719:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176690.232722:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176693.304693:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176693.304697:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176693.304699:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1786176694.777164:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176694.777169:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176694.777928:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176694.777932:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176694.778209:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176694.778210:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176694.778412:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176694.778414:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176694.778751:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176694.778753:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:2.0:1786176696.376646:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176696.376650:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176696.376652:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176699.448799:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176699.448802:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176699.448805:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176699.896720:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176699.896727:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176699.896730:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176699.896734:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176699.896738:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176699.896745:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6923
00000400:00000200:2.0:1786176699.896754:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176699.896761:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176699.896765:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176699.896768:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.896772:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176699.896775:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.896777:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176699.896781:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176699.896783:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176699.896787:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176699.896790:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176699.896793:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176699.896795:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176699.896799:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.896801:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176699.896803:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.896806:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176699.896808:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176699.896812:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176699.896815:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176699.896818:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176699.896821:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176699.896825:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.896827:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176699.896830:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176699.896838:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786176699.896842:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176699.896847:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176699.896849:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.896852:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176699.896855:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176699.896860:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176699.896869:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786176699.896876:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176699.896886:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856286592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176699.896896:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176699.896901:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176699.896907:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856286592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176699.896931:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176699.896935:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176699.896938:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176699.896940:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176699.896942:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176699.896944:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176699.896947:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6923
00000400:00000200:2.0:1786176699.896951:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176699.896955:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176699.896957:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176699.896961:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.896963:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176699.896965:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.896967:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176699.896969:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176699.896983:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176699.896988:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176699.896992:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176699.896997:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176699.897002:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176699.897008:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897014:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176699.897019:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897025:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176699.897029:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176699.897036:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897045:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176699.897050:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176699.897057:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176699.897062:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.897067:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176699.897073:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176699.897078:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.897083:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176699.897091:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000020:00000040:2.0:1786176699.897095:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176699.897099:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176699.897103:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.897108:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176699.897110:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176699.897113:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176699.897120:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786176699.897126:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176699.897136:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856286720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176699.897150:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176699.897156:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176699.897166:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856286720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176699.897177:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176699.897183:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176699.897192:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176699.897196:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176699.897199:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176699.897203:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176699.897210:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6923
00000400:00000200:2.0:1786176699.897218:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176699.897225:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176699.897230:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176699.897238:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897243:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176699.897248:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897255:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176699.897258:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176699.897259:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176699.897261:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176699.897263:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176699.897265:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176699.897269:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176699.897277:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897282:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176699.897287:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897291:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176699.897296:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176699.897303:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897309:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176699.897315:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176699.897321:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176699.897328:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.897332:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176699.897334:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176699.897340:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.897346:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176699.897353:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000020:00000040:2.0:1786176699.897359:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176699.897365:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176699.897367:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.897370:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176699.897374:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176699.897378:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176699.897385:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786176699.897391:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176699.897398:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:2.0:1786176699.897400:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856286848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1786176699.897405:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1786176699.897426:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176699.897431:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176699.897441:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856286848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176699.897457:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176699.897462:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176699.897469:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176699.897472:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176699.897476:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176699.897491:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176699.897498:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6923
00000400:00000200:2.0:1786176699.897508:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176699.897555:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176699.897561:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176699.897581:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897587:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176699.897591:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897597:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176699.897600:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176699.897601:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176699.897603:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176699.897608:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176699.897676:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176699.897683:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176699.897693:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897700:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176699.897706:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897711:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176699.897719:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176699.897727:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897734:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176699.897738:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176699.897741:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176699.897744:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.897747:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176699.897751:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176699.897758:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.897762:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176699.897770:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786176699.897776:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176699.897782:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176699.897785:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.897789:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176699.897793:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176699.897797:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176699.897803:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786176699.897810:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176699.897821:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856286976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176699.897839:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176699.897842:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176699.897850:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856286976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176699.897870:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176699.897876:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176699.897883:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176699.897887:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176699.897890:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176699.897894:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176699.897901:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6923
00000400:00000200:2.0:1786176699.897910:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176699.897917:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176699.897923:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176699.897929:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897933:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176699.897938:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897944:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176699.897950:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176699.897955:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176699.897959:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176699.897964:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176699.897969:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176699.897974:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176699.897981:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897987:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176699.897993:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176699.897998:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176699.898003:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786176699.898007:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176699.898012:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176699.898017:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176699.898023:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176699.898028:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.898033:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176699.898036:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176699.898043:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.898047:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176699.898054:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786176699.898059:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176699.898066:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176699.898069:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176699.898075:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176699.898078:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176699.898083:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176699.898091:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786176699.898098:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176699.898110:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856287104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176699.898130:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176699.898134:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176699.898141:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856287104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176699.898160:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176699.898168:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6933)
00000800:00000200:3.1:1786176699.898251:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176699.898254:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176699.898632:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176699.898635:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176699.898954:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176699.898957:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176699.899382:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176699.899384:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:2.0:1786176702.520801:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176702.520806:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176702.520809:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.1:1786176705.017146:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176705.017153:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:0.0:1786176705.017493:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.017497:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176705.017500:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176705.017505:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176705.017563:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176705.017567:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176705.017574:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856287232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176705.017589:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856287232:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176705.017596:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.017599:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176705.017607:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856287232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.017627:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176705.017630:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176705.017633:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176705.017639:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176705.017643:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176705.017645:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176705.017649:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176705.017652:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786176705.017657:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176705.017660:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176705.017662:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000001:0.0:1786176705.017667:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176705.017669:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.017672:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.017678:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176705.017681:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176705.017683:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176705.017685:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176705.017686:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176705.017690:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856287360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.017700:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176705.017701:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176705.017703:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176705.017704:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176705.017705:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176705.017707:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856287360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176705.017709:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176705.017711:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176705.017712:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176705.017713:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176705.017716:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176705.017717:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176705.017720:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176705.017722:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176705.017725:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176705.017727:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856287360, portal 10
00000100:00000001:0.0:1786176705.017728:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176705.017729:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000040:0.0:1786176705.017734:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856287360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.017738:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176705.017740:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176705.017742:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176705.017743:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856287360, offset 0
00000400:00000200:0.0:1786176705.017746:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176705.017749:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176705.017751:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176705.017753:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176705.017755:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176705.017757:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176705.017758:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176705.017761:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254416:254416:256:4294967295] 192.168.202.110@tcp LPNI seq info [254405:254405:8:4294967295]
00000400:00000200:0.0:1786176705.017764:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176705.017768:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176705.017772:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176705.017774:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176705.017777:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176705.017779:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176705.017782:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176705.017784:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1786176705.017787:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176705.017789:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176705.017791:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176705.017793:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176705.017795:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176705.017797:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176705.017798:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176705.017801:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856287360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.017806:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176705.017808:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856287360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.017812:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176705.017813:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176705.017816:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176705.017818:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176705.017820:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176705.017821:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301600778
00000100:00000001:0.0:1786176705.017823:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301604557
00000100:00000001:0.0:1786176705.017825:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301600778
00000100:00000001:0.0:1786176705.017827:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301604557
00000100:00000001:0.0:1786176705.017828:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176705.017829:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176705.017831:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176705.017833:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176705/real 1786176705]  req@ffff97d1ba69fb80 x1872934856287360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176705.017838:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176705.017839:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176705.017840:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176705.017842:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176705.017843:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176705.017844:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176705.017846:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856287360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176705.017852:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856287360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.017856:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176705.017857:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.017858:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176705.017858:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176705.017861:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856287360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.017865:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176705.017867:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856287360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.017871:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176705.017872:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.017873:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176705.017874:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.017875:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176705.017876:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176705.017878:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176705.017880:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176705.017882:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176705.017885:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856287360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176705.017889:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856287360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176705.017892:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176705.017894:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856287488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.017898:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176705.017898:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176705.017900:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176705.017901:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176705.017902:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176705.017904:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856287488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176705.017905:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176705.017906:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176705.017907:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176705.017908:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176705.017910:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176705.017911:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176705.017913:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176705.017914:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176705.017916:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176705.017918:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856287488, portal 10
00000100:00000001:0.0:1786176705.017919:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176705.017920:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:0.0:1786176705.017924:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856287488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.017928:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176705.017930:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176705.017932:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176705.017933:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856287488, offset 0
00000400:00000200:0.0:1786176705.017936:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176705.017938:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176705.017940:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176705.017942:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176705.017943:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176705.017945:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176705.017946:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176705.017948:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254417:254417:256:4294967295] 192.168.202.110@tcp LPNI seq info [254406:254406:8:4294967295]
00000400:00000200:0.0:1786176705.017951:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176705.017955:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176705.017958:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176705.017960:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176705.017962:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176705.017964:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176705.017966:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176705.017968:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1786176705.017971:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176705.017972:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176705.017974:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176705.017976:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176705.017978:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176705.017979:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176705.017980:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176705.017982:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856287488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.017988:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176705.017990:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856287488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.017994:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176705.017995:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176705.017997:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176705.017998:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176705.018001:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176705.018002:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301600903
00000100:00000001:0.0:1786176705.018004:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301604557
00000100:00000001:0.0:1786176705.018005:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301600903
00000100:00000001:0.0:1786176705.018006:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301604557
00000100:00000001:0.0:1786176705.018007:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176705.018009:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176705.018010:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176705.018012:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176705/real 1786176705]  req@ffff97d1ba69ca80 x1872934856287488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176705.018016:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176705.018017:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176705.018018:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176705.018019:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176705.018020:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176705.018021:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176705.018023:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856287488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176705.018029:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856287488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018032:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176705.018033:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018034:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176705.018035:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176705.018037:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856287488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018041:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176705.018044:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856287488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018049:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176705.018050:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018051:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176705.018052:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018053:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176705.018054:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176705.018056:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176705.018058:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176705.018059:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176705.018061:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856287488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176705.018066:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856287488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176705.018068:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176705.018070:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856287616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018074:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176705.018075:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176705.018076:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176705.018077:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176705.018078:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176705.018080:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856287616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176705.018082:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176705.018083:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176705.018084:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176705.018085:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176705.018087:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176705.018088:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176705.018090:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176705.018092:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176705.018094:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176705.018096:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856287616, portal 4
00000100:00000001:0.0:1786176705.018097:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176705.018098:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:0.0:1786176705.018102:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856287616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018107:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176705.018109:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176705.018110:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176705.018112:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856287616, offset 0
00000400:00000200:0.0:1786176705.018115:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176705.018117:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176705.018120:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176705.018123:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176705.018126:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176705.018128:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176705.018130:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176705.018133:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254418:254418:256:4294967295] 192.168.202.110@tcp LPNI seq info [254407:254407:8:4294967295]
00000400:00000200:0.0:1786176705.018137:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176705.018143:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176705.018149:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176705.018152:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176705.018156:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176705.018159:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176705.018163:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176705.018166:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1786176705.018170:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176705.018172:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176705.018175:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176705.018177:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176705.018179:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176705.018181:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176705.018183:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176705.018186:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856287616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018191:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176705.018193:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856287616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018198:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176705.018199:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176705.018201:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176705.018203:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176705.018205:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176705.018207:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301600778
00000100:00000001:0.0:1786176705.018208:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301604557
00000100:00000001:0.0:1786176705.018210:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301600778
00000100:00000001:0.0:1786176705.018211:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301604557
00000100:00000001:0.0:1786176705.018213:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176705.018214:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176705.018215:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176705.018217:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176705/real 1786176705]  req@ffff97d1ba69f480 x1872934856287616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176705.018222:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176705.018223:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176705.018224:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176705.018226:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176705.018227:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176705.018228:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176705.018231:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856287616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176705.018236:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856287616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018240:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176705.018242:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018243:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176705.018244:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176705.018246:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856287616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018250:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176705.018252:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856287616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018256:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176705.018258:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018259:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176705.018259:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018260:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176705.018261:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176705.018263:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176705.018265:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176705.018266:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176705.018268:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856287616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176705.018273:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856287616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176705.018275:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176705.018277:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856287744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018281:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176705.018282:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176705.018284:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176705.018285:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176705.018287:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176705.018289:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856287744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176705.018291:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176705.018292:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176705.018293:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176705.018294:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176705.018296:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176705.018297:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176705.018299:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176705.018301:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176705.018303:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176705.018305:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856287744, portal 4
00000100:00000001:0.0:1786176705.018306:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176705.018307:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:0.0:1786176705.018311:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856287744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018315:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176705.018317:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176705.018318:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176705.018319:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856287744, offset 0
00000400:00000200:0.0:1786176705.018322:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176705.018324:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176705.018326:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176705.018328:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176705.018330:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176705.018331:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176705.018333:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176705.018335:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254419:254419:256:4294967295] 192.168.202.110@tcp LPNI seq info [254408:254408:8:4294967295]
00000400:00000200:0.0:1786176705.018338:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176705.018342:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176705.018345:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176705.018347:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176705.018350:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176705.018351:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176705.018354:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176705.018356:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1786176705.018359:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176705.018360:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176705.018362:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176705.018364:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176705.018366:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176705.018367:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176705.018368:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176705.018371:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856287744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018375:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176705.018377:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856287744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018381:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176705.018382:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176705.018384:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176705.018386:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176705.018388:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176705.018389:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301600786
00000100:00000001:0.0:1786176705.018391:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301604557
00000100:00000001:0.0:1786176705.018392:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301600786
00000100:00000001:0.0:1786176705.018393:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301604557
00000100:00000001:0.0:1786176705.018395:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176705.018396:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176705.018397:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176705.018399:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176705/real 1786176705]  req@ffff97d1ba69f100 x1872934856287744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176705.018405:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176705.018406:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176705.018407:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176705.018408:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176705.018409:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176705.018410:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176705.018412:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856287744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176705.018417:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856287744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018421:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176705.018422:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018423:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176705.018424:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176705.018426:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856287744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018430:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176705.018432:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856287744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018436:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176705.018438:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018439:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176705.018440:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018441:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176705.018442:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176705.018443:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176705.018445:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176705.018446:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176705.018449:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856287744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176705.018454:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856287744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176705.018456:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018458:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176705.018459:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856287360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018463:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176705.018464:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176705.018465:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176705.018467:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176705.018468:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176705.018469:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176705.018470:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176705.018471:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:0.0:1786176705.018472:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176705.018473:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176705.018474:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:0.0:1786176705.018475:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176705.018476:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018477:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176705.018479:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856287488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018484:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176705.018484:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176705.018485:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176705.018486:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176705.018487:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176705.018488:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176705.018489:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176705.018490:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786176705.018491:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176705.018492:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176705.018493:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:0.0:1786176705.018494:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176705.018495:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018496:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176705.018498:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856287616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018502:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176705.018503:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176705.018504:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176705.018505:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176705.018506:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176705.018506:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176705.018507:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176705.018536:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786176705.018539:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176705.018540:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176705.018541:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:0.0:1786176705.018542:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176705.018543:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018544:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176705.018547:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856287744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176730 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176705.018551:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176705.018552:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176705.018553:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176705.018554:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176705.018554:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176705.018555:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176705.018557:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176705.018557:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786176705.018559:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176705.018559:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176705.018560:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:0.0:1786176705.018561:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176705.018562:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018564:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176705.018566:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176705.018567:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176705.018569:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176705.018570:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176705.592722:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176705.592726:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176705.592729:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176706.616749:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176706.616753:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176706.616757:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176708.664735:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176708.664739:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176708.664741:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176708.856622:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301604557, at 4301604608
00000100:00000010:0.0:1786176708.856642:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000020:00000001:0.0:1786176709.688687:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176709.688689:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176709.688692:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786176710.136866:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176710.136870:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176710.136872:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176710.136879:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856287872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.136896:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176710.136898:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176710.136904:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176710.136910:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176710.136915:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176710.136923:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856287872:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176710.136930:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176710.136934:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176710.136939:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176710.136945:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176710.136954:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176710.136958:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176710.136965:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176710.136970:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176710.136981:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176710.136991:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856287872, portal 25
00000100:00000001:0.0:1786176710.136995:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176710.136999:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000100:00000040:0.0:1786176710.137009:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856287872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137019:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176710.137027:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176710.137030:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176710.137034:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856287872, offset 0
00000400:00000200:0.0:1786176710.137043:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176710.137053:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176710.137062:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176710.137072:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176710.137078:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176710.137084:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176710.137089:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176710.137094:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254420:254420:256:4294967295] 192.168.202.110@tcp LPNI seq info [254409:254409:8:4294967295]
00000400:00000200:0.0:1786176710.137103:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176710.137112:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176710.137120:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176710.137126:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176710.137134:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176710.137138:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176710.137146:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176710.137151:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1786176710.137160:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176710.137162:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176710.137169:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176710.137177:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176710.137184:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176710.137188:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176710.137195:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176710.137201:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856287872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137215:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176710.137221:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856287872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137230:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.137232:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176710.137238:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176710.137244:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176710.137248:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301609677
00000100:00000010:0.0:1786176710.137254:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176710.137259:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176710.137263:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301609677
00000100:00000010:0.0:1786176710.137267:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134780.
00000100:00000001:0.0:1786176710.137271:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176710.137276:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176710.137281:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.137283:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176710.137289:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176710/real 1786176710]  req@ffff97d1ba69d500 x1872934856287872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176710.137302:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176710.137306:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176710.137310:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176710.137314:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176710.137317:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176710.137319:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176710.137321:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856287872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176710.137334:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856287872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137342:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176710.137344:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.137345:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176710.137347:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176710.137353:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856287872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137366:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176710.137374:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856287872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137391:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176710.137396:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.137400:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176710.137402:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.137406:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176710.137410:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176710.137415:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176710.137423:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176710.137429:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176710.137434:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856287872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176710.137440:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856287872:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176710.137445:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.137449:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176710.137453:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856287872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137465:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176710.137487:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176710.137491:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176710.137496:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176710.137499:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176710.137502:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176710.137505:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176710.137508:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786176710.137534:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176710.137537:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176710.137540:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:0.0:1786176710.137543:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176710.137544:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.137545:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.137550:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176710.137552:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.137554:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176710.137557:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176710.137558:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176710.137566:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856288000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137581:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176710.137584:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176710.137587:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176710.137592:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176710.137597:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176710.137602:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176710.137606:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176710.137609:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176710.137612:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176710.137616:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176710.137620:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176710.137624:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176710.137631:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176710.137636:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176710.137640:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176710.137644:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856288000, portal 10
00000100:00000001:0.0:1786176710.137650:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176710.137653:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:0.0:1786176710.137663:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856288000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137678:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176710.137686:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176710.137691:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176710.137697:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856288000, offset 0
00000400:00000200:0.0:1786176710.137706:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176710.137712:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176710.137717:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176710.137723:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176710.137727:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176710.137732:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176710.137736:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176710.137748:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254421:254421:256:4294967295] 192.168.202.110@tcp LPNI seq info [254410:254410:8:4294967295]
00000400:00000200:0.0:1786176710.137761:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176710.137773:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176710.137785:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176710.137789:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176710.137798:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176710.137803:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176710.137811:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176710.137815:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1786176710.137820:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176710.137821:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176710.137824:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176710.137826:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176710.137829:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176710.137830:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176710.137833:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176710.137836:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856288000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137842:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176710.137844:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856288000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137848:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.137849:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176710.137852:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176710.137855:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176710.137857:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176710.137859:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301605898
00000100:00000001:0.0:1786176710.137861:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301609677
00000100:00000001:0.0:1786176710.137862:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301605898
00000100:00000001:0.0:1786176710.137864:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301609677
00000100:00000001:0.0:1786176710.137865:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176710.137866:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.137868:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176710.137870:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176710/real 1786176710]  req@ffff97d1ba69ea00 x1872934856288000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176710.137875:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176710.137875:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176710.137876:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176710.137878:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176710.137879:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176710.137880:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176710.137882:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856288000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176710.137887:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856288000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137891:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176710.137892:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.137893:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176710.137894:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176710.137896:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856288000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137901:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176710.137905:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856288000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137913:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176710.137916:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.137918:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176710.137919:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.137921:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176710.137922:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176710.137925:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176710.137929:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176710.137931:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176710.137936:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856288000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176710.137944:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176710.137949:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176710.137952:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856288128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.137959:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176710.137960:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176710.137962:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176710.137964:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176710.137966:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176710.137970:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176710.137973:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176710.137975:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176710.137976:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176710.137978:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176710.137981:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176710.137983:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176710.137986:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176710.137987:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176710.137990:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176710.137992:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856288128, portal 10
00000100:00000001:0.0:1786176710.137993:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176710.137994:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:0.0:1786176710.137998:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856288128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138003:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176710.138005:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176710.138006:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176710.138007:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856288128, offset 0
00000400:00000200:0.0:1786176710.138010:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176710.138012:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176710.138015:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176710.138017:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176710.138018:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176710.138020:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176710.138022:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176710.138024:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254422:254422:256:4294967295] 192.168.202.110@tcp LPNI seq info [254411:254411:8:4294967295]
00000400:00000200:0.0:1786176710.138027:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176710.138031:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176710.138034:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176710.138036:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176710.138038:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176710.138040:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176710.138043:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176710.138045:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1786176710.138048:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176710.138049:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176710.138051:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176710.138053:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176710.138055:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176710.138057:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176710.138058:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176710.138061:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856288128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138066:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176710.138068:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856288128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138072:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.138073:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176710.138075:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176710.138076:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176710.138078:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176710.138080:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301606023
00000100:00000001:0.0:1786176710.138081:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301609677
00000100:00000001:0.0:1786176710.138082:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301606023
00000100:00000001:0.0:1786176710.138084:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301609677
00000100:00000001:0.0:1786176710.138085:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176710.138086:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.138088:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176710.138090:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176710/real 1786176710]  req@ffff97d1ba69c000 x1872934856288128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176710.138095:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176710.138096:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176710.138097:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176710.138098:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176710.138099:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176710.138101:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176710.138103:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856288128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176710.138108:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856288128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138112:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176710.138113:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138114:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176710.138115:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176710.138118:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856288128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138121:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176710.138124:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856288128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138128:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176710.138129:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138130:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176710.138131:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138132:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176710.138133:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176710.138135:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176710.138137:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176710.138138:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176710.138140:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856288128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176710.138145:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176710.138148:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138149:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176710.138151:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856288000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138155:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176710.138156:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176710.138158:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176710.138159:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176710.138161:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176710.138162:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176710.138163:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176710.138164:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786176710.138166:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176710.138167:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176710.138168:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:0.0:1786176710.138170:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176710.138171:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138172:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176710.138174:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856288128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138178:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176710.138179:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176710.138179:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176710.138181:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176710.138181:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176710.138182:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176710.138183:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176710.138184:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786176710.138185:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176710.138186:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176710.138187:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:0.0:1786176710.138188:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176710.138189:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138190:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138192:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176710.138193:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.138194:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176710.138195:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176710.138196:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176710.138198:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856288256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138202:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176710.138203:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176710.138205:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176710.138206:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176710.138207:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176710.138209:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176710.138211:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176710.138212:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176710.138213:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176710.138214:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176710.138216:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176710.138217:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176710.138219:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176710.138220:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176710.138222:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176710.138224:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856288256, portal 4
00000100:00000001:0.0:1786176710.138225:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176710.138226:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:0.0:1786176710.138230:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856288256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138235:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176710.138236:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176710.138237:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176710.138239:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856288256, offset 0
00000400:00000200:0.0:1786176710.138241:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176710.138243:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176710.138245:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176710.138247:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176710.138248:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176710.138250:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176710.138252:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176710.138254:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254423:254423:256:4294967295] 192.168.202.110@tcp LPNI seq info [254412:254412:8:4294967295]
00000400:00000200:0.0:1786176710.138256:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176710.138260:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176710.138263:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176710.138265:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176710.138268:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176710.138270:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176710.138272:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176710.138274:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1786176710.138277:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176710.138278:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176710.138280:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176710.138282:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176710.138284:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176710.138285:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176710.138287:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176710.138289:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856288256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138294:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176710.138296:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856288256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138300:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.138300:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176710.138302:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176710.138304:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176710.138306:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176710.138307:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301605898
00000100:00000001:0.0:1786176710.138309:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301609677
00000100:00000001:0.0:1786176710.138310:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301605898
00000100:00000001:0.0:1786176710.138311:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301609677
00000100:00000001:0.0:1786176710.138313:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176710.138314:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.138315:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176710.138317:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176710/real 1786176710]  req@ffff97d1ba69c700 x1872934856288256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176710.138323:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176710.138324:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176710.138325:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176710.138326:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176710.138327:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176710.138328:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176710.138330:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856288256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176710.138335:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856288256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138339:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176710.138340:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138341:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176710.138342:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176710.138344:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856288256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138348:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176710.138350:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856288256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138354:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176710.138356:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138357:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176710.138358:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138359:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176710.138360:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176710.138361:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176710.138363:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176710.138364:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176710.138367:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856288256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176710.138372:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176710.138374:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176710.138376:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856288384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138380:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176710.138381:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176710.138382:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176710.138383:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176710.138384:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176710.138386:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176710.138388:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176710.138389:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176710.138390:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176710.138391:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176710.138392:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176710.138393:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176710.138395:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176710.138396:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176710.138398:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176710.138400:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856288384, portal 4
00000100:00000001:0.0:1786176710.138401:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176710.138402:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:0.0:1786176710.138405:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856288384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138409:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176710.138411:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176710.138412:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176710.138414:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856288384, offset 0
00000400:00000200:0.0:1786176710.138417:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176710.138419:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176710.138421:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176710.138422:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176710.138424:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176710.138426:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176710.138427:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176710.138429:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254424:254424:256:4294967295] 192.168.202.110@tcp LPNI seq info [254413:254413:8:4294967295]
00000400:00000200:0.0:1786176710.138431:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176710.138435:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176710.138438:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176710.138439:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176710.138441:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176710.138443:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176710.138446:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176710.138447:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1786176710.138450:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176710.138451:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176710.138453:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176710.138455:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176710.138457:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176710.138458:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176710.138459:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176710.138461:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856288384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138466:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176710.138483:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856288384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176735 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138488:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.138489:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176710.138492:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176710.138494:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176710.138496:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176710.138498:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301605906
00000100:00000001:0.0:1786176710.138499:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301609677
00000100:00000001:0.0:1786176710.138501:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301605906
00000100:00000001:0.0:1786176710.138502:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301609677
00000100:00000001:0.0:1786176710.138503:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176710.138505:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.138506:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176710.138508:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176710/real 1786176710]  req@ffff97d1ba69e300 x1872934856288384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176710.138528:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176710.138529:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176710.138530:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176710.138532:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176710.138533:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176710.138534:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176710.138537:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856288384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176710.138542:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856288384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138546:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176710.138547:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138548:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176710.138549:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176710.138552:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856288384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138556:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176710.138559:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856288384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138563:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176710.138565:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138566:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176710.138567:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138568:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176710.138569:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176710.138571:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176710.138573:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176710.138574:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176710.138576:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856288384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176710.138581:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176710.138583:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138584:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176710.138586:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856288256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138591:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176710.138592:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176710.138593:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176710.138595:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176710.138596:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176710.138597:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176710.138598:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176710.138600:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786176710.138602:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176710.138602:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176710.138603:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:0.0:1786176710.138605:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176710.138605:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138607:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176710.138609:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856288384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176735 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176710.138617:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176710.138618:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176710.138619:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176710.138620:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176710.138621:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176710.138622:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176710.138623:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176710.138623:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786176710.138625:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176710.138625:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176710.138626:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:0.0:1786176710.138627:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176710.138628:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138629:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176710.138631:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176710.138632:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176710.138633:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176710.138634:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176711.736677:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176711.736681:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176711.736684:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176712.760694:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176712.760697:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176712.760701:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176713.976648:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301609677, at 4301609728
00000100:00000010:0.0:1786176713.976662:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000020:00000001:2.0:1786176714.808710:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176714.808714:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176714.808718:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786176715.256898:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176715.256904:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176715.256908:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176715.256918:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d180 x1872934856288512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.256932:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176715.256934:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176715.256938:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176715.256941:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176715.256949:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176715.256953:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856288512:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176715.256959:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176715.256963:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176715.256966:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176715.256969:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176715.256977:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176715.256981:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176715.256988:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176715.256992:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176715.256997:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176715.257008:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856288512, portal 25
00000100:00000001:0.0:1786176715.257013:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176715.257016:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297088 : -114547945254528 : ffff97d1ba69d180)
00000100:00000040:0.0:1786176715.257023:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d180 x1872934856288512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.257033:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176715.257038:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176715.257041:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176715.257045:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856288512, offset 0
00000400:00000200:0.0:1786176715.257054:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176715.257065:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176715.257072:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176715.257079:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176715.257086:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176715.257092:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176715.257097:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176715.257111:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254425:254425:256:4294967295] 192.168.202.110@tcp LPNI seq info [254414:254414:8:4294967295]
00000400:00000200:0.0:1786176715.257124:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176715.257144:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176715.257158:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176715.257167:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176715.257179:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176715.257187:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176715.257199:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176715.257207:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786176715.257217:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176715.257224:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176715.257233:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176715.257243:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176715.257253:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176715.257258:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176715.257266:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176715.257274:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d180 x1872934856288512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.257291:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176715.257299:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d180 x1872934856288512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.257316:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.257321:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176715.257328:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176715.257337:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176715.257343:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301614797
00000100:00000010:0.0:1786176715.257353:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176715.257360:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176715.257367:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301614797
00000100:00000010:0.0:1786176715.257371:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134d80.
00000100:00000001:0.0:1786176715.257376:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176715.257381:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176715.257385:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.257390:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176715.257396:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176715/real 1786176715]  req@ffff97d1ba69d180 x1872934856288512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176715.257411:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176715.257415:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176715.257433:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176715.257437:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176715.257440:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176715.257444:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176715.257449:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d180 x1872934856288512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176715.257465:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d180 x1872934856288512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.257477:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176715.257480:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.257485:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176715.257487:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176715.257494:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d180 x1872934856288512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.257505:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176715.257542:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d180 x1872934856288512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.257558:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176715.257562:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.257565:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176715.257568:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.257572:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176715.257576:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176715.257580:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176715.257586:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176715.257591:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176715.257598:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d180 x1872934856288512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176715.257611:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856288512:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176715.257623:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.257628:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176715.257632:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d180 x1872934856288512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.257644:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176715.257647:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176715.257650:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176715.257654:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176715.257658:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176715.257661:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176715.257663:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176715.257665:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:0.0:1786176715.257671:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176715.257672:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176715.257675:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d180.
00000100:00000001:0.0:1786176715.257679:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176715.257682:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.257684:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.257691:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176715.257695:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.257699:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176715.257700:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176715.257702:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176715.257705:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856288640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.257715:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176715.257719:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176715.257723:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176715.257728:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176715.257733:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176715.257742:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176715.257751:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176715.257756:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176715.257759:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176715.257765:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176715.257770:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176715.257774:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176715.257783:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176715.257788:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176715.257792:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176715.257798:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856288640, portal 10
00000100:00000001:0.0:1786176715.257803:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176715.257806:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:0.0:1786176715.257816:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856288640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.257827:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176715.257832:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176715.257836:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176715.257840:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856288640, offset 0
00000400:00000200:0.0:1786176715.257850:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176715.257860:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176715.257869:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176715.257875:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176715.257882:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176715.257889:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176715.257895:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176715.257904:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254426:254426:256:4294967295] 192.168.202.110@tcp LPNI seq info [254415:254415:8:4294967295]
00000400:00000200:0.0:1786176715.257915:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176715.257933:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176715.257948:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176715.257954:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176715.257964:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176715.257969:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176715.257981:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176715.257986:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786176715.257996:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176715.258001:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176715.258009:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176715.258018:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176715.258026:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176715.258031:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176715.258037:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176715.258044:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856288640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258063:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176715.258068:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856288640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258077:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.258080:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176715.258084:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176715.258090:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176715.258097:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176715.258099:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301611018
00000100:00000001:0.0:1786176715.258104:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301614797
00000100:00000001:0.0:1786176715.258107:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301611018
00000100:00000001:0.0:1786176715.258113:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301614797
00000100:00000001:0.0:1786176715.258115:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176715.258116:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.258118:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176715.258122:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176715/real 1786176715]  req@ffff97d1ba69e300 x1872934856288640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176715.258127:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176715.258129:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176715.258130:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176715.258132:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176715.258134:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176715.258136:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176715.258139:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856288640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176715.258145:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856288640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258149:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176715.258150:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258151:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176715.258152:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176715.258155:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856288640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258159:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176715.258161:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856288640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258165:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176715.258167:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258168:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176715.258169:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258170:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176715.258172:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176715.258173:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176715.258175:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176715.258177:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176715.258179:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856288640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176715.258184:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176715.258187:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176715.258189:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856288768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258198:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176715.258199:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176715.258201:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176715.258202:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176715.258204:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176715.258205:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176715.258207:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176715.258208:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176715.258210:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176715.258211:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176715.258213:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176715.258215:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176715.258217:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176715.258219:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176715.258221:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176715.258223:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856288768, portal 10
00000100:00000001:0.0:1786176715.258224:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176715.258225:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:0.0:1786176715.258229:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856288768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258233:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176715.258235:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176715.258236:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176715.258238:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856288768, offset 0
00000400:00000200:0.0:1786176715.258240:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176715.258243:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176715.258245:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176715.258247:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176715.258249:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176715.258251:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176715.258252:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176715.258255:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254427:254427:256:4294967295] 192.168.202.110@tcp LPNI seq info [254416:254416:8:4294967295]
00000400:00000200:0.0:1786176715.258258:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176715.258262:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176715.258265:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176715.258268:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176715.258270:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176715.258272:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176715.258275:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176715.258277:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786176715.258280:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176715.258281:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176715.258283:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176715.258285:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176715.258287:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176715.258289:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176715.258290:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176715.258292:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856288768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258297:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176715.258299:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856288768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258304:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.258305:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176715.258306:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176715.258308:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176715.258310:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176715.258311:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301611143
00000100:00000001:0.0:1786176715.258312:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301614797
00000100:00000001:0.0:1786176715.258314:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301611143
00000100:00000001:0.0:1786176715.258315:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301614797
00000100:00000001:0.0:1786176715.258317:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176715.258318:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.258320:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176715.258322:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176715/real 1786176715]  req@ffff97d1ba69c700 x1872934856288768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176715.258327:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176715.258328:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176715.258329:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176715.258330:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176715.258331:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176715.258332:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176715.258334:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856288768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176715.258340:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856288768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258344:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176715.258345:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258346:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176715.258347:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176715.258349:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856288768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258353:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176715.258355:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856288768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258359:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176715.258361:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258361:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176715.258362:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258363:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176715.258364:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176715.258366:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176715.258368:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176715.258370:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176715.258372:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856288768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176715.258377:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176715.258379:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258381:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176715.258382:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856288640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258386:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176715.258387:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176715.258389:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176715.258390:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176715.258391:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176715.258392:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176715.258393:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176715.258394:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786176715.258396:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176715.258396:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176715.258397:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:0.0:1786176715.258399:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176715.258400:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258401:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176715.258403:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856288768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258407:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176715.258408:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176715.258409:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176715.258410:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176715.258411:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176715.258412:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176715.258413:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176715.258414:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786176715.258415:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176715.258416:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176715.258432:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:0.0:1786176715.258436:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176715.258437:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258438:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258441:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176715.258442:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.258443:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176715.258444:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176715.258445:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176715.258448:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856288896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258453:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176715.258454:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176715.258455:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176715.258457:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176715.258458:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176715.258460:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176715.258462:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176715.258464:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176715.258465:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176715.258466:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176715.258468:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176715.258469:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176715.258472:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176715.258474:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176715.258476:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176715.258478:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856288896, portal 4
00000100:00000001:0.0:1786176715.258479:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176715.258480:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:0.0:1786176715.258485:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856288896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258494:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176715.258497:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176715.258499:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176715.258501:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856288896, offset 0
00000400:00000200:0.0:1786176715.258505:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176715.258529:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176715.258534:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176715.258537:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176715.258540:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176715.258542:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176715.258544:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176715.258547:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254428:254428:256:4294967295] 192.168.202.110@tcp LPNI seq info [254417:254417:8:4294967295]
00000400:00000200:0.0:1786176715.258551:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176715.258556:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176715.258560:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176715.258563:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176715.258566:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176715.258567:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176715.258570:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176715.258572:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786176715.258574:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176715.258576:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176715.258578:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176715.258581:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176715.258583:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176715.258584:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176715.258586:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176715.258589:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856288896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258594:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176715.258596:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856288896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258605:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.258606:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176715.258608:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176715.258610:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176715.258612:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176715.258614:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301611019
00000100:00000001:0.0:1786176715.258616:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301614797
00000100:00000001:0.0:1786176715.258617:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301611019
00000100:00000001:0.0:1786176715.258619:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301614797
00000100:00000001:0.0:1786176715.258620:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176715.258622:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.258623:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176715.258625:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176715/real 1786176715]  req@ffff97d1ba69c000 x1872934856288896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176715.258630:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176715.258631:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176715.258632:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176715.258634:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176715.258635:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176715.258636:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176715.258638:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856288896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176715.258644:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856288896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258647:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176715.258648:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258649:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176715.258650:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176715.258652:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856288896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258657:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176715.258660:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856288896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258664:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176715.258666:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258667:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176715.258668:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258669:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176715.258670:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176715.258671:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176715.258673:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176715.258675:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176715.258677:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856288896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176715.258681:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856288896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176715.258684:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176715.258686:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856289024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258690:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176715.258690:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176715.258692:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176715.258693:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176715.258695:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176715.258696:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856289024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176715.258698:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176715.258700:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176715.258701:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176715.258702:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176715.258704:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176715.258705:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176715.258707:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176715.258709:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176715.258712:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176715.258715:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856289024, portal 4
00000100:00000001:0.0:1786176715.258717:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176715.258718:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:0.0:1786176715.258724:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856289024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258731:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176715.258733:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176715.258735:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176715.258738:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856289024, offset 0
00000400:00000200:0.0:1786176715.258742:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176715.258746:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176715.258750:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176715.258753:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176715.258756:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176715.258758:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176715.258761:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176715.258764:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254429:254429:256:4294967295] 192.168.202.110@tcp LPNI seq info [254418:254418:8:4294967295]
00000400:00000200:0.0:1786176715.258768:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176715.258772:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176715.258775:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176715.258778:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176715.258781:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176715.258783:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176715.258785:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176715.258787:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786176715.258790:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176715.258792:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176715.258794:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176715.258796:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176715.258798:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176715.258799:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176715.258801:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176715.258804:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856289024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258809:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176715.258811:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856289024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258815:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.258816:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176715.258818:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176715.258821:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176715.258823:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176715.258824:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301611027
00000100:00000001:0.0:1786176715.258826:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301614797
00000100:00000001:0.0:1786176715.258827:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301611027
00000100:00000001:0.0:1786176715.258829:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301614797
00000100:00000001:0.0:1786176715.258830:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176715.258831:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.258832:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176715.258835:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176715/real 1786176715]  req@ffff97d1ba69ea00 x1872934856289024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176715.258840:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176715.258840:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176715.258842:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176715.258845:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176715.258847:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176715.258849:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176715.258853:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856289024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176715.258861:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856289024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258868:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176715.258869:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258871:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176715.258873:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176715.258877:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856289024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258885:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176715.258889:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856289024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258896:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176715.258898:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258899:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176715.258900:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258901:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176715.258902:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176715.258904:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176715.258906:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176715.258908:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176715.258910:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856289024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176715.258916:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856289024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176715.258918:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258919:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176715.258921:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856288896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258926:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176715.258927:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176715.258928:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176715.258930:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176715.258931:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176715.258932:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176715.258933:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176715.258935:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786176715.258937:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176715.258937:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176715.258939:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:0.0:1786176715.258940:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176715.258941:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258942:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176715.258944:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856289024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176715.258948:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176715.258949:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176715.258950:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176715.258951:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176715.258952:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176715.258953:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176715.258954:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176715.258955:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786176715.258956:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176715.258957:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176715.258957:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:0.0:1786176715.258959:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176715.258959:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258960:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176715.258963:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176715.258964:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176715.258965:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176715.258966:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176715.832707:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176715.832711:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176715.832714:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176717.880684:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176717.880689:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176717.880692:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176718.904710:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176718.904714:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176718.904717:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176719.096623:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301614797, at 4301614848
00000100:00000010:0.0:1786176719.096638:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176720.376823:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176720.376829:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176720.376833:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176720.376842:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856289152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.376858:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176720.376861:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176720.376867:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176720.376871:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176720.376877:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176720.376883:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856289152:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176720.376888:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176720.376892:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176720.376895:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176720.376898:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176720.376905:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176720.376908:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176720.376916:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176720.376922:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176720.376928:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176720.376946:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856289152, portal 25
00000100:00000001:0.0:1786176720.376951:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176720.376954:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000100:00000040:0.0:1786176720.376963:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856289152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.376983:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176720.376987:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176720.376990:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176720.376994:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856289152, offset 0
00000400:00000200:0.0:1786176720.377005:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176720.377016:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176720.377025:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176720.377032:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176720.377039:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176720.377045:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176720.377050:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176720.377056:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254430:254430:256:4294967295] 192.168.202.110@tcp LPNI seq info [254419:254419:8:4294967295]
00000400:00000200:0.0:1786176720.377066:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176720.377078:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176720.377089:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176720.377097:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176720.377108:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176720.377114:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176720.377125:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176720.377131:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786176720.377140:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176720.377144:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176720.377151:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176720.377160:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176720.377167:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176720.377170:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176720.377178:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176720.377185:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856289152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377203:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176720.377209:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856289152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377231:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176720.377237:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176720.377243:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176720.377250:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176720.377254:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301619917
00000100:00000010:0.0:1786176720.377259:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176720.377267:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176720.377275:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301619917
00000100:00000010:0.0:1786176720.377280:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1851340c0.
00000100:00000001:0.0:1786176720.377284:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176720.377288:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176720.377292:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176720.377296:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176720.377301:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176720/real 1786176720]  req@ffff97d1ba69d500 x1872934856289152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176720.377314:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176720.377316:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176720.377318:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176720.377322:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176720.377325:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176720.377328:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176720.377334:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856289152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176720.377346:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856289152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377357:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176720.377360:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.377366:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176720.377391:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176720.377401:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856289152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377419:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176720.377426:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856289152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377440:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176720.377447:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.377451:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176720.377454:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.377457:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176720.377461:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176720.377468:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176720.377475:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176720.377480:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176720.377486:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856289152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176720.377504:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856289152:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176720.377563:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.377566:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176720.377574:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856289152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377591:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176720.377596:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176720.377601:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176720.377606:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176720.377610:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176720.377612:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176720.377616:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176720.377619:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786176720.377623:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176720.377625:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176720.377627:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:0.0:1786176720.377629:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176720.377630:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.377632:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.377639:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176720.377640:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176720.377641:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176720.377642:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176720.377644:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176720.377647:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856289280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377652:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176720.377653:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176720.377655:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176720.377657:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176720.377658:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176720.377660:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856289280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176720.377662:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176720.377663:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176720.377665:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176720.377666:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176720.377668:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176720.377670:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176720.377673:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176720.377674:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176720.377677:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176720.377679:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856289280, portal 10
00000100:00000001:0.0:1786176720.377681:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176720.377682:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:0.0:1786176720.377686:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856289280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377691:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176720.377692:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176720.377694:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176720.377696:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856289280, offset 0
00000400:00000200:0.0:1786176720.377698:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176720.377701:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176720.377704:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176720.377707:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176720.377708:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176720.377710:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176720.377712:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176720.377714:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254431:254431:256:4294967295] 192.168.202.110@tcp LPNI seq info [254420:254420:8:4294967295]
00000400:00000200:0.0:1786176720.377717:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176720.377721:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176720.377725:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176720.377727:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176720.377730:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176720.377731:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176720.377734:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176720.377736:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786176720.377739:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176720.377740:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176720.377742:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176720.377744:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176720.377746:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176720.377748:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176720.377750:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176720.377752:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856289280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377757:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176720.377759:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856289280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377764:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176720.377765:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176720.377767:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176720.377770:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176720.377772:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176720.377774:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301616138
00000100:00000001:0.0:1786176720.377776:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301619917
00000100:00000001:0.0:1786176720.377777:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301616138
00000100:00000001:0.0:1786176720.377779:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301619917
00000100:00000001:0.0:1786176720.377780:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176720.377781:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176720.377782:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176720.377784:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176720/real 1786176720]  req@ffff97d1ba69f100 x1872934856289280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176720.377790:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176720.377791:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176720.377792:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176720.377794:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176720.377795:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176720.377796:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176720.377798:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856289280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176720.377803:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856289280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377807:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176720.377808:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.377809:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176720.377810:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176720.377812:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856289280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377816:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176720.377819:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856289280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377824:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176720.377826:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.377827:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176720.377828:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.377829:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176720.377830:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176720.377832:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176720.377833:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176720.377835:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176720.377837:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856289280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176720.377842:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856289280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176720.377844:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176720.377846:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856289408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377851:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176720.377852:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176720.377854:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176720.377855:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176720.377856:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176720.377858:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856289408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176720.377859:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176720.377860:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176720.377861:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176720.377862:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176720.377864:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176720.377866:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176720.377868:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176720.377869:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176720.377871:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176720.377873:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856289408, portal 10
00000100:00000001:0.0:1786176720.377874:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176720.377875:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:0.0:1786176720.377879:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856289408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377883:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176720.377884:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176720.377886:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176720.377887:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856289408, offset 0
00000400:00000200:0.0:1786176720.377890:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176720.377892:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176720.377893:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176720.377895:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176720.377897:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176720.377898:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176720.377900:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176720.377902:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254432:254432:256:4294967295] 192.168.202.110@tcp LPNI seq info [254421:254421:8:4294967295]
00000400:00000200:0.0:1786176720.377905:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176720.377909:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176720.377911:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176720.377913:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176720.377915:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176720.377917:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176720.377919:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176720.377921:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786176720.377923:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176720.377925:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176720.377926:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176720.377929:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176720.377931:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176720.377932:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176720.377934:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176720.377936:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856289408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377941:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176720.377943:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856289408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377947:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176720.377948:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176720.377949:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176720.377951:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176720.377953:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176720.377954:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301616263
00000100:00000001:0.0:1786176720.377956:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301619917
00000100:00000001:0.0:1786176720.377958:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301616263
00000100:00000001:0.0:1786176720.377959:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301619917
00000100:00000001:0.0:1786176720.377960:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176720.377961:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176720.377962:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176720.377965:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176720/real 1786176720]  req@ffff97d1ba69f480 x1872934856289408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176720.377969:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176720.377970:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176720.377971:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176720.377972:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176720.377973:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176720.377974:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176720.377976:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856289408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176720.377982:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856289408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377986:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176720.377987:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.377988:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176720.377989:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176720.377992:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856289408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.377995:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176720.377998:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856289408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378002:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176720.378003:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378004:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176720.378005:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378006:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176720.378007:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176720.378008:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176720.378010:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176720.378011:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176720.378014:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856289408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176720.378018:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856289408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176720.378020:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176720.378022:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856289536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378027:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176720.378028:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176720.378029:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176720.378031:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176720.378032:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176720.378034:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856289536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176720.378036:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176720.378037:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176720.378038:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176720.378039:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176720.378041:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176720.378042:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176720.378044:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176720.378045:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176720.378047:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176720.378048:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856289536, portal 4
00000100:00000001:0.0:1786176720.378050:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176720.378051:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:0.0:1786176720.378055:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856289536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378059:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176720.378060:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176720.378061:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176720.378063:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856289536, offset 0
00000400:00000200:0.0:1786176720.378065:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176720.378067:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176720.378069:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176720.378071:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176720.378072:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176720.378074:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176720.378076:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176720.378078:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254433:254433:256:4294967295] 192.168.202.110@tcp LPNI seq info [254422:254422:8:4294967295]
00000400:00000200:0.0:1786176720.378080:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176720.378084:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176720.378087:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176720.378089:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176720.378091:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176720.378093:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176720.378096:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176720.378097:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786176720.378100:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176720.378101:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176720.378103:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176720.378105:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176720.378106:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176720.378107:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176720.378109:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176720.378111:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856289536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378115:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176720.378117:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856289536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378122:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176720.378123:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176720.378124:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176720.378126:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176720.378127:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176720.378128:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301616138
00000100:00000001:0.0:1786176720.378130:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301619917
00000100:00000001:0.0:1786176720.378131:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301616138
00000100:00000001:0.0:1786176720.378133:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301619917
00000100:00000001:0.0:1786176720.378134:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176720.378135:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176720.378136:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176720.378139:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176720/real 1786176720]  req@ffff97d1ba69ca80 x1872934856289536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176720.378144:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176720.378145:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176720.378146:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176720.378147:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176720.378148:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176720.378149:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176720.378151:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856289536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176720.378156:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856289536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378160:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176720.378161:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378162:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176720.378163:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176720.378165:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856289536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378169:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176720.378171:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856289536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378175:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176720.378177:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378177:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176720.378178:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378179:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176720.378180:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176720.378182:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176720.378183:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176720.378185:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176720.378187:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856289536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176720.378192:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856289536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176720.378195:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176720.378197:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856289664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378200:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176720.378201:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176720.378202:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176720.378203:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176720.378205:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176720.378206:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856289664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176720.378208:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176720.378209:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176720.378210:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176720.378211:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176720.378213:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176720.378214:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176720.378215:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176720.378217:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176720.378218:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176720.378220:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856289664, portal 4
00000100:00000001:0.0:1786176720.378221:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176720.378222:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000040:0.0:1786176720.378225:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856289664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378229:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176720.378231:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176720.378232:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176720.378234:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856289664, offset 0
00000400:00000200:0.0:1786176720.378236:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176720.378238:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176720.378240:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176720.378242:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176720.378243:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176720.378245:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176720.378247:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176720.378249:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254434:254434:256:4294967295] 192.168.202.110@tcp LPNI seq info [254423:254423:8:4294967295]
00000400:00000200:0.0:1786176720.378251:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176720.378255:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176720.378258:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176720.378259:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176720.378262:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176720.378263:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176720.378266:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176720.378268:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786176720.378270:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176720.378271:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176720.378273:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176720.378275:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176720.378277:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176720.378278:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176720.378279:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176720.378281:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856289664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378286:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176720.378288:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856289664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378292:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176720.378292:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176720.378294:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176720.378296:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176720.378297:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176720.378299:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301616146
00000100:00000001:0.0:1786176720.378301:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301619917
00000100:00000001:0.0:1786176720.378302:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301616146
00000100:00000001:0.0:1786176720.378303:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301619917
00000100:00000001:0.0:1786176720.378304:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176720.378306:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176720.378307:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176720.378309:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176720/real 1786176720]  req@ffff97d1ba69fb80 x1872934856289664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176720.378313:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176720.378314:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176720.378315:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176720.378316:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176720.378317:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176720.378318:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176720.378320:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856289664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176720.378326:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856289664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378329:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176720.378330:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378331:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176720.378332:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176720.378334:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856289664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378338:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176720.378340:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856289664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378344:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176720.378346:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378347:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176720.378348:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378349:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176720.378350:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176720.378352:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176720.378354:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176720.378355:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176720.378357:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856289664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176720.378362:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856289664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176720.378364:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378365:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176720.378367:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856289280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378388:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176720.378389:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176720.378391:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176720.378392:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176720.378393:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176720.378394:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176720.378395:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176720.378397:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786176720.378398:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176720.378399:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176720.378400:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:0.0:1786176720.378402:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176720.378403:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378404:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176720.378407:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856289408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378416:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176720.378416:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176720.378418:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176720.378419:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176720.378420:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176720.378420:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176720.378422:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176720.378422:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786176720.378424:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176720.378424:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176720.378425:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:0.0:1786176720.378426:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176720.378427:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378428:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176720.378430:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856289536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378433:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176720.378434:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176720.378435:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176720.378436:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176720.378437:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176720.378438:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176720.378439:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176720.378439:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786176720.378441:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176720.378442:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176720.378442:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:0.0:1786176720.378444:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176720.378445:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378445:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176720.378447:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856289664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176720.378451:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176720.378452:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176720.378453:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176720.378454:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176720.378455:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176720.378456:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176720.378457:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176720.378458:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786176720.378459:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176720.378460:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176720.378461:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:0.0:1786176720.378462:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176720.378463:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378464:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176720.378466:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176720.378467:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176720.378468:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176720.378469:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176720.952681:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176720.952696:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176720.952700:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176721.976700:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176721.976704:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176721.976706:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.0:1786176722.296884:0:2384:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000020:00000001:2.0:1786176724.024680:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176724.024683:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176724.024686:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176724.216720:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301619917, at 4301619968
00000100:00000010:0.0:1786176724.216725:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000020:00000001:0.0:1786176725.048761:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176725.048767:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176725.048771:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786176725.496914:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176725.496920:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176725.496923:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176725.496933:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856289792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.496947:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176725.496949:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176725.496954:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176725.496958:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176725.496962:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176725.496969:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856289792:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176725.496978:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176725.496982:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176725.496985:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176725.496988:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176725.496993:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176725.496997:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176725.497003:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176725.497018:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176725.497026:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176725.497038:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856289792, portal 25
00000100:00000001:0.0:1786176725.497043:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176725.497047:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
00000100:00000040:0.0:1786176725.497059:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856289792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497074:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176725.497080:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176725.497083:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176725.497089:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856289792, offset 0
00000400:00000200:0.0:1786176725.497101:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176725.497112:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176725.497123:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176725.497131:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176725.497143:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176725.497152:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176725.497158:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176725.497167:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254435:254435:256:4294967295] 192.168.202.110@tcp LPNI seq info [254424:254424:8:4294967295]
00000400:00000200:0.0:1786176725.497180:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176725.497199:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176725.497207:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176725.497214:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176725.497225:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176725.497231:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176725.497237:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176725.497243:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786176725.497249:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176725.497253:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176725.497258:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176725.497265:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176725.497271:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176725.497273:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176725.497278:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176725.497285:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f800 x1872934856289792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497301:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176725.497308:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856289792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497338:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176725.497342:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176725.497351:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176725.497360:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176725.497365:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301625037
00000100:00000010:0.0:1786176725.497372:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176725.497380:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176725.497388:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301625037
00000100:00000010:0.0:1786176725.497394:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134600.
00000100:00000001:0.0:1786176725.497405:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176725.497410:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176725.497415:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176725.497421:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176725.497432:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176725/real 1786176725]  req@ffff97d1ba69f800 x1872934856289792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176725.497457:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176725.497461:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176725.497465:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176725.497470:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176725.497473:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176725.497478:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176725.497486:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856289792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176725.497507:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856289792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497586:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176725.497591:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.497598:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176725.497604:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176725.497615:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856289792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497637:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176725.497648:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856289792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497681:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176725.497688:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.497692:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176725.497693:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.497696:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176725.497699:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176725.497701:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176725.497706:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176725.497708:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176725.497712:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856289792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176725.497719:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856289792:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176725.497725:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.497727:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176725.497729:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856289792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497740:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176725.497743:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176725.497745:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176725.497748:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176725.497751:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176725.497752:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176725.497755:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176725.497757:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786176725.497761:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176725.497762:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176725.497764:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000001:0.0:1786176725.497767:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176725.497768:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.497771:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.497779:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176725.497781:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176725.497783:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176725.497785:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176725.497786:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176725.497791:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856289920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497799:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176725.497800:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176725.497802:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176725.497803:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176725.497805:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176725.497807:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856289920:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176725.497809:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176725.497811:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176725.497812:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176725.497814:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176725.497816:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176725.497818:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176725.497821:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176725.497823:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176725.497825:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176725.497828:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856289920, portal 10
00000100:00000001:0.0:1786176725.497829:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176725.497830:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
00000100:00000040:0.0:1786176725.497835:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856289920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497840:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176725.497841:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176725.497843:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176725.497844:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856289920, offset 0
00000400:00000200:0.0:1786176725.497847:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176725.497849:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176725.497852:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176725.497854:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176725.497856:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176725.497858:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176725.497859:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176725.497862:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254436:254436:256:4294967295] 192.168.202.110@tcp LPNI seq info [254425:254425:8:4294967295]
00000400:00000200:0.0:1786176725.497865:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176725.497869:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176725.497872:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176725.497874:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176725.497877:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176725.497879:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176725.497882:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176725.497883:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786176725.497886:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176725.497887:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176725.497890:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176725.497892:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176725.497894:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176725.497896:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176725.497897:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176725.497899:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856289920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497905:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176725.497907:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856289920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497911:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176725.497912:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176725.497914:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176725.497918:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176725.497920:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176725.497921:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301621258
00000100:00000001:0.0:1786176725.497923:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301625037
00000100:00000001:0.0:1786176725.497924:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301621258
00000100:00000001:0.0:1786176725.497926:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301625037
00000100:00000001:0.0:1786176725.497927:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176725.497928:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176725.497929:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176725.497932:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176725/real 1786176725]  req@ffff97d1ba69d880 x1872934856289920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176725.497937:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176725.497938:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176725.497939:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176725.497941:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176725.497942:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176725.497943:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176725.497946:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856289920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176725.497951:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856289920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497955:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176725.497956:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.497957:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176725.497958:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176725.497960:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856289920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497964:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176725.497967:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856289920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.497971:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176725.497972:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.497973:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176725.497974:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.497975:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176725.497976:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176725.497978:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176725.497980:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176725.497981:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176725.497983:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856289920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176725.497988:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856289920:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176725.497991:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176725.497993:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856290048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498002:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176725.498003:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176725.498005:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176725.498006:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176725.498007:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176725.498009:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176725.498010:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176725.498011:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176725.498012:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176725.498013:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176725.498015:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176725.498016:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176725.498018:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176725.498019:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176725.498021:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176725.498023:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856290048, portal 10
00000100:00000001:0.0:1786176725.498024:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176725.498025:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:0.0:1786176725.498029:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856290048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498033:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176725.498034:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176725.498036:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176725.498038:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856290048, offset 0
00000400:00000200:0.0:1786176725.498040:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176725.498042:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176725.498044:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176725.498046:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176725.498048:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176725.498049:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176725.498051:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176725.498053:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254437:254437:256:4294967295] 192.168.202.110@tcp LPNI seq info [254426:254426:8:4294967295]
00000400:00000200:0.0:1786176725.498056:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176725.498060:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176725.498063:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176725.498065:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176725.498067:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176725.498068:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176725.498071:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176725.498073:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786176725.498075:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176725.498076:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176725.498078:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176725.498080:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176725.498082:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176725.498083:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176725.498085:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176725.498087:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856290048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498092:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176725.498094:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856290048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498098:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176725.498099:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176725.498101:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176725.498103:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176725.498104:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176725.498106:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301621383
00000100:00000001:0.0:1786176725.498107:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301625037
00000100:00000001:0.0:1786176725.498109:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301621383
00000100:00000001:0.0:1786176725.498110:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301625037
00000100:00000001:0.0:1786176725.498111:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176725.498113:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176725.498114:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176725.498116:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176725/real 1786176725]  req@ffff97d1ba69ed80 x1872934856290048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176725.498121:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176725.498122:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176725.498123:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176725.498124:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176725.498125:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176725.498126:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176725.498128:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856290048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176725.498133:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856290048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498138:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176725.498139:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498141:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176725.498142:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176725.498146:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856290048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498153:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176725.498157:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856290048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498164:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176725.498167:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498169:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176725.498170:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498172:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176725.498174:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176725.498177:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176725.498181:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176725.498183:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176725.498188:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856290048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176725.498196:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176725.498199:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176725.498201:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856290176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498205:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176725.498207:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176725.498208:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176725.498210:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176725.498211:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176725.498212:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176725.498214:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176725.498215:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176725.498216:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176725.498217:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176725.498220:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176725.498221:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176725.498224:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176725.498226:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176725.498228:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176725.498230:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856290176, portal 4
00000100:00000001:0.0:1786176725.498232:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176725.498233:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00000040:0.0:1786176725.498238:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856290176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498242:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176725.498244:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176725.498245:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176725.498247:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856290176, offset 0
00000400:00000200:0.0:1786176725.498249:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176725.498252:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176725.498254:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176725.498257:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176725.498258:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176725.498260:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176725.498261:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176725.498264:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254438:254438:256:4294967295] 192.168.202.110@tcp LPNI seq info [254427:254427:8:4294967295]
00000400:00000200:0.0:1786176725.498266:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176725.498270:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176725.498273:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176725.498274:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176725.498277:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176725.498279:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176725.498281:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176725.498283:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786176725.498286:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176725.498287:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176725.498289:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176725.498291:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176725.498293:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176725.498294:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176725.498296:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176725.498298:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856290176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498304:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176725.498306:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856290176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498310:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176725.498311:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176725.498313:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176725.498315:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176725.498317:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176725.498333:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301621258
00000100:00000001:0.0:1786176725.498335:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301625037
00000100:00000001:0.0:1786176725.498337:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301621258
00000100:00000001:0.0:1786176725.498338:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301625037
00000100:00000001:0.0:1786176725.498340:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176725.498341:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176725.498342:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176725.498345:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176725/real 1786176725]  req@ffff97d1ba69e680 x1872934856290176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176725.498351:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176725.498352:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176725.498353:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176725.498355:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176725.498356:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176725.498357:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176725.498359:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856290176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176725.498364:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856290176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498369:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176725.498370:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498371:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176725.498372:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176725.498374:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856290176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498378:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176725.498380:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856290176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498385:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176725.498387:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498388:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176725.498389:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498390:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176725.498391:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176725.498392:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176725.498394:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176725.498395:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176725.498397:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856290176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176725.498402:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176725.498404:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176725.498407:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69dc00 x1872934856290304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498414:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176725.498415:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176725.498418:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176725.498419:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176725.498421:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176725.498424:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176725.498427:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176725.498429:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176725.498431:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176725.498433:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176725.498437:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176725.498439:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176725.498443:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176725.498446:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176725.498449:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176725.498453:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856290304, portal 4
00000100:00000001:0.0:1786176725.498455:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176725.498456:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764299776 : -114547945251840 : ffff97d1ba69dc00)
00000100:00000040:0.0:1786176725.498462:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69dc00 x1872934856290304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498466:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176725.498468:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176725.498469:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176725.498471:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856290304, offset 0
00000400:00000200:0.0:1786176725.498474:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176725.498476:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176725.498479:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176725.498481:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176725.498483:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176725.498484:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176725.498486:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176725.498488:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254439:254439:256:4294967295] 192.168.202.110@tcp LPNI seq info [254428:254428:8:4294967295]
00000400:00000200:0.0:1786176725.498490:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176725.498494:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176725.498497:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176725.498499:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176725.498501:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176725.498503:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176725.498506:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176725.498507:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786176725.498527:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176725.498529:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176725.498531:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176725.498533:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176725.498536:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176725.498537:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176725.498539:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176725.498541:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69dc00 x1872934856290304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498547:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176725.498549:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69dc00 x1872934856290304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498553:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176725.498554:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176725.498556:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176725.498558:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176725.498560:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176725.498562:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301621267
00000100:00000001:0.0:1786176725.498563:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301625037
00000100:00000001:0.0:1786176725.498565:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301621267
00000100:00000001:0.0:1786176725.498566:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301625037
00000100:00000001:0.0:1786176725.498568:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176725.498569:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176725.498570:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176725.498572:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176725/real 1786176725]  req@ffff97d1ba69dc00 x1872934856290304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176725.498577:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176725.498578:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176725.498579:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176725.498580:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176725.498581:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176725.498582:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176725.498584:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69dc00 x1872934856290304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176725.498589:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69dc00 x1872934856290304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498598:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176725.498599:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498600:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176725.498601:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176725.498603:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69dc00 x1872934856290304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498607:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176725.498609:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69dc00 x1872934856290304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498613:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176725.498615:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498616:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176725.498616:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498617:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176725.498619:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176725.498620:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176725.498622:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176725.498623:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176725.498625:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69dc00 x1872934856290304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176725.498630:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176725.498633:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498634:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176725.498636:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856289920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498640:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176725.498641:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176725.498643:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176725.498644:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176725.498645:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176725.498646:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176725.498648:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176725.498649:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:0.0:1786176725.498651:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176725.498652:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176725.498653:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:0.0:1786176725.498654:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176725.498655:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498656:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176725.498659:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856290048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498663:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176725.498664:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176725.498665:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176725.498666:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176725.498667:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176725.498668:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176725.498669:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176725.498670:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786176725.498671:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176725.498671:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176725.498672:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:0.0:1786176725.498674:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176725.498674:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498675:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176725.498677:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856290176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498681:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176725.498681:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176725.498682:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176725.498683:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176725.498684:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176725.498685:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176725.498686:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176725.498687:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786176725.498689:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176725.498689:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176725.498690:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:0.0:1786176725.498692:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176725.498692:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498693:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176725.498695:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69dc00 x1872934856290304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176725.498699:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176725.498699:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176725.498700:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176725.498701:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176725.498702:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176725.498702:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176725.498703:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176725.498704:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786176725.498705:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176725.498706:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176725.498707:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69dc00.
00000100:00000001:0.0:1786176725.498708:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176725.498709:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498710:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176725.498712:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176725.498713:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176725.498714:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176725.498715:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176727.096670:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176727.096673:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176727.096676:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176728.120719:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176728.120723:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176728.120727:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176729.336600:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301625037, at 4301625088
00000100:00000010:0.0:1786176729.336625:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000020:00000001:2.0:1786176730.168752:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176730.168758:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176730.168761:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786176730.616843:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176730.616849:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176730.616854:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176730.616866:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c380 x1872934856290432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.616897:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176730.616912:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176730.616919:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176730.616925:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176730.616932:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176730.616941:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856290432:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176730.616951:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176730.616958:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176730.616962:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176730.616965:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176730.616973:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176730.616978:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176730.616989:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176730.616997:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176730.617008:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176730.617019:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856290432, portal 25
00000100:00000001:0.0:1786176730.617026:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176730.617031:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764293504 : -114547945258112 : ffff97d1ba69c380)
00000100:00000040:0.0:1786176730.617045:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c380 x1872934856290432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.617069:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176730.617078:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176730.617085:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176730.617092:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856290432, offset 0
00000400:00000200:0.0:1786176730.617116:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176730.617130:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176730.617142:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176730.617155:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176730.617164:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176730.617173:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176730.617182:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176730.617201:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254440:254440:256:4294967295] 192.168.202.110@tcp LPNI seq info [254429:254429:8:4294967295]
00000400:00000200:0.0:1786176730.617215:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176730.617245:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176730.617260:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176730.617283:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176730.617297:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176730.617307:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176730.617318:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176730.617326:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1786176730.617337:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176730.617342:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176730.617349:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176730.617361:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176730.617368:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176730.617373:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176730.617381:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176730.617389:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c380 x1872934856290432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.617415:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176730.617425:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c380 x1872934856290432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.617446:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176730.617450:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176730.617460:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176730.617472:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176730.617479:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301630157
00000100:00000010:0.0:1786176730.617489:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176730.617500:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176730.617545:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301630157
00000100:00000010:0.0:1786176730.617554:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134ea0.
00000100:00000001:0.0:1786176730.617560:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176730.617567:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176730.617574:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176730.617579:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176730.617587:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176730/real 1786176730]  req@ffff97d1ba69c380 x1872934856290432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176730.617607:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176730.617620:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176730.617624:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176730.617631:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176730.617635:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176730.617640:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176730.617649:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c380 x1872934856290432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176730.617666:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c380 x1872934856290432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.617674:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176730.617676:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.617678:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176730.617680:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176730.617685:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c380 x1872934856290432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.617692:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176730.617697:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c380 x1872934856290432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.617705:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176730.617708:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.617710:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176730.617711:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.617713:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176730.617715:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176730.617719:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176730.617723:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176730.617726:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176730.617730:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c380 x1872934856290432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176730.617745:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856290432:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176730.617750:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.617752:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176730.617756:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c380 x1872934856290432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.617764:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176730.617766:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176730.617769:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176730.617773:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176730.617775:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176730.617777:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176730.617779:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176730.617782:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786176730.617798:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176730.617800:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176730.617801:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c380.
00000100:00000001:0.0:1786176730.617805:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176730.617806:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.617809:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.617815:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176730.617817:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176730.617819:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176730.617820:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176730.617822:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176730.617826:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69dc00 x1872934856290560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.617832:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176730.617834:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176730.617837:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176730.617839:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176730.617841:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176730.617844:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176730.617848:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176730.617850:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176730.617852:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176730.617865:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176730.617868:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176730.617870:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176730.617874:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176730.617877:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176730.617880:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176730.617883:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856290560, portal 10
00000100:00000001:0.0:1786176730.617885:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176730.617887:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764299776 : -114547945251840 : ffff97d1ba69dc00)
00000100:00000040:0.0:1786176730.617893:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69dc00 x1872934856290560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.617899:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176730.617902:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176730.617904:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176730.617906:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856290560, offset 0
00000400:00000200:0.0:1786176730.617911:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176730.617915:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176730.617919:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176730.617922:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176730.617925:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176730.617927:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176730.617930:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176730.617933:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254441:254441:256:4294967295] 192.168.202.110@tcp LPNI seq info [254430:254430:8:4294967295]
00000400:00000200:0.0:1786176730.617937:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176730.617943:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176730.617959:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176730.617962:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176730.617967:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176730.617970:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176730.617974:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176730.617977:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1786176730.617981:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176730.617983:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176730.617986:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176730.617990:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176730.617994:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176730.617995:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176730.617998:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176730.618001:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69dc00 x1872934856290560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618009:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176730.618013:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69dc00 x1872934856290560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618020:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176730.618021:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176730.618024:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176730.618028:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176730.618031:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176730.618034:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301626378
00000100:00000001:0.0:1786176730.618036:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301630157
00000100:00000001:0.0:1786176730.618038:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301626378
00000100:00000001:0.0:1786176730.618041:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301630157
00000100:00000001:0.0:1786176730.618043:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176730.618045:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176730.618047:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176730.618051:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176730/real 1786176730]  req@ffff97d1ba69dc00 x1872934856290560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176730.618065:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176730.618066:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176730.618068:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176730.618070:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176730.618072:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176730.618074:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176730.618078:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69dc00 x1872934856290560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176730.618086:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69dc00 x1872934856290560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618093:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176730.618094:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.618096:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176730.618098:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176730.618101:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69dc00 x1872934856290560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618108:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176730.618112:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69dc00 x1872934856290560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618118:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176730.618121:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.618122:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176730.618124:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.618126:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176730.618128:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176730.618131:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176730.618134:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176730.618136:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176730.618140:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69dc00 x1872934856290560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176730.618148:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176730.618152:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176730.618155:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856290688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618162:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176730.618164:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176730.618166:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176730.618168:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176730.618170:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176730.618173:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176730.618176:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176730.618178:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176730.618179:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176730.618181:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176730.618184:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176730.618187:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176730.618190:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176730.618193:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176730.618196:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176730.618198:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856290688, portal 10
00000100:00000001:0.0:1786176730.618201:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176730.618212:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00000040:0.0:1786176730.618218:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856290688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618225:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176730.618227:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176730.618229:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176730.618231:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856290688, offset 0
00000400:00000200:0.0:1786176730.618236:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176730.618240:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176730.618244:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176730.618246:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176730.618249:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176730.618252:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176730.618254:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176730.618257:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254442:254442:256:4294967295] 192.168.202.110@tcp LPNI seq info [254431:254431:8:4294967295]
00000400:00000200:0.0:1786176730.618261:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176730.618268:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176730.618288:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176730.618292:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176730.618296:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176730.618298:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176730.618303:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176730.618305:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1786176730.618309:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176730.618312:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176730.618315:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176730.618318:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176730.618322:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176730.618324:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176730.618326:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176730.618330:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856290688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618338:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176730.618341:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856290688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618348:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176730.618350:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176730.618352:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176730.618355:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176730.618358:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176730.618361:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301626503
00000100:00000001:0.0:1786176730.618364:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301630157
00000100:00000001:0.0:1786176730.618366:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301626503
00000100:00000001:0.0:1786176730.618369:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301630157
00000100:00000001:0.0:1786176730.618371:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176730.618373:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176730.618375:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176730.618379:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176730/real 1786176730]  req@ffff97d1ba69e680 x1872934856290688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176730.618387:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176730.618388:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176730.618390:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176730.618392:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176730.618394:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176730.618396:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176730.618400:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856290688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176730.618408:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856290688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618415:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176730.618416:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.618418:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176730.618421:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176730.618425:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856290688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618432:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176730.618436:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856290688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618442:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176730.618445:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.618447:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176730.618448:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.618450:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176730.618452:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176730.618455:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176730.618458:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176730.618460:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176730.618463:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856290688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176730.618471:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176730.618475:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176730.618478:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856290816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618486:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176730.618487:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176730.618489:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176730.618491:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176730.618493:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176730.618496:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176730.618500:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176730.618501:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176730.618503:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176730.618505:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176730.618508:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176730.618529:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176730.618533:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176730.618537:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176730.618540:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176730.618543:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856290816, portal 4
00000100:00000001:0.0:1786176730.618545:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176730.618547:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:0.0:1786176730.618553:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856290816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618560:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176730.618562:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176730.618565:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176730.618567:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856290816, offset 0
00000400:00000200:0.0:1786176730.618571:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176730.618575:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176730.618579:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176730.618592:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176730.618595:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176730.618597:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176730.618600:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176730.618603:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254443:254443:256:4294967295] 192.168.202.110@tcp LPNI seq info [254432:254432:8:4294967295]
00000400:00000200:0.0:1786176730.618607:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176730.618613:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176730.618618:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176730.618621:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176730.618625:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176730.618628:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176730.618632:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176730.618634:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1786176730.618638:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176730.618640:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176730.618643:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176730.618647:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176730.618650:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176730.618651:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176730.618654:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176730.618657:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856290816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618671:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176730.618675:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856290816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618681:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176730.618682:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176730.618685:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176730.618688:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176730.618691:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176730.618693:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301626379
00000100:00000001:0.0:1786176730.618696:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301630157
00000100:00000001:0.0:1786176730.618698:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301626379
00000100:00000001:0.0:1786176730.618700:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301630157
00000100:00000001:0.0:1786176730.618703:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176730.618705:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176730.618707:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176730.618711:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176730/real 1786176730]  req@ffff97d1ba69ed80 x1872934856290816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176730.618718:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176730.618719:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176730.618721:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176730.618724:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176730.618725:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176730.618727:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176730.618731:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856290816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176730.618739:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856290816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618746:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176730.618748:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.618750:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176730.618751:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176730.618755:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856290816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618762:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176730.618766:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856290816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618774:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176730.618777:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.618778:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176730.618780:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.618782:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176730.618784:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176730.618786:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176730.618789:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176730.618792:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176730.618796:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856290816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176730.618804:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176730.618808:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176730.618812:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856290944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618818:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176730.618819:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176730.618821:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176730.618823:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176730.618825:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176730.618827:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176730.618830:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176730.618832:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176730.618834:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176730.618836:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176730.618839:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176730.618841:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176730.618845:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176730.618848:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176730.618851:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176730.618854:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856290944, portal 4
00000100:00000001:0.0:1786176730.618855:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176730.618857:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
00000100:00000040:0.0:1786176730.618863:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856290944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618869:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176730.618871:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176730.618873:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176730.618876:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856290944, offset 0
00000400:00000200:0.0:1786176730.618880:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176730.618884:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176730.618888:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176730.618891:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176730.618893:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176730.618896:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176730.618898:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176730.618901:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254444:254444:256:4294967295] 192.168.202.110@tcp LPNI seq info [254433:254433:8:4294967295]
00000400:00000200:0.0:1786176730.618905:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176730.618911:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176730.618916:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176730.618919:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176730.618924:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176730.618927:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1786176730.618931:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176730.618933:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1786176730.618938:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176730.618940:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:0.0:1786176730.618943:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176730.618946:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176730.618949:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:0.0:1786176730.618951:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:0.0:1786176730.618953:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176730.618957:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856290944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618964:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176730.618968:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856290944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.618974:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176730.618976:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176730.618979:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176730.618982:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176730.618985:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176730.618988:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301626387
00000100:00000001:0.0:1786176730.618991:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301630157
00000100:00000001:0.0:1786176730.618993:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301626387
00000100:00000001:0.0:1786176730.618995:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301630157
00000100:00000001:0.0:1786176730.618997:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176730.618999:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176730.619001:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176730.619005:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176730/real 1786176730]  req@ffff97d1ba69d880 x1872934856290944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176730.619013:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176730.619015:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176730.619016:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176730.619019:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176730.619021:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176730.619023:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176730.619026:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856290944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176730.619035:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856290944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.619041:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176730.619042:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.619044:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176730.619046:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176730.619050:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856290944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.619056:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176730.619060:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856290944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.619067:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176730.619069:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.619071:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176730.619072:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.619074:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176730.619076:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176730.619079:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176730.619082:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176730.619084:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176730.619088:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856290944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176730.619102:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856290944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176730.619106:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.619108:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176730.619111:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69dc00 x1872934856290560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.619118:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176730.619120:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176730.619122:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176730.619124:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176730.619126:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176730.619127:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176730.619130:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176730.619131:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786176730.619134:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176730.619136:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176730.619137:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69dc00.
00000100:00000001:0.0:1786176730.619140:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176730.619142:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.619143:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176730.619147:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856290688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.619154:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176730.619155:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176730.619156:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176730.619159:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176730.619161:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176730.619162:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176730.619164:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176730.619165:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786176730.619168:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176730.619169:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176730.619170:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:0.0:1786176730.619173:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176730.619174:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.619176:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176730.619180:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856290816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.619186:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176730.619187:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176730.619189:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176730.619191:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176730.619192:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176730.619193:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176730.619195:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176730.619197:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786176730.619199:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176730.619200:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176730.619202:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:0.0:1786176730.619204:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176730.619205:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.619207:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176730.619210:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856290944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176730.619216:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176730.619218:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176730.619219:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176730.619221:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176730.619222:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176730.619223:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176730.619225:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176730.619226:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786176730.619229:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176730.619230:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176730.619232:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:0.0:1786176730.619234:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176730.619235:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.619237:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176730.619240:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176730.619242:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176730.619244:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176730.619246:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176731.193924:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176731.193929:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176731.193932:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176733.240731:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176733.240735:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176733.240739:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176734.264729:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176734.264733:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176734.264737:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176734.456580:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301630157, at 4301630208
00000100:00000010:0.0:1786176734.456594:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176735.736858:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176735.736864:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176735.736869:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176735.736879:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856291072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176735.736897:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176735.736899:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176735.736916:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176735.736920:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176735.736923:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176735.736926:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856291072:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176735.736933:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176735.736937:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176735.736940:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176735.736943:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176735.736990:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176735.736995:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176735.737004:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176735.737011:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786176735.737019:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786176735.737027:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856291072, portal 25
00000100:00000001:0.0:1786176735.737034:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176735.737037:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
00000100:00000040:0.0:1786176735.737048:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856291072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176735.737063:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176735.737069:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176735.737073:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4f68.
00000100:00000200:0.0:1786176735.737077:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856291072, offset 0
00000400:00000200:0.0:1786176735.737086:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176735.737096:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176735.737103:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176735.737109:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176735.737116:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176735.737122:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176735.737127:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176735.737135:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254445:254445:256:4294967295] 192.168.202.110@tcp LPNI seq info [254434:254434:8:4294967295]
00000400:00000200:0.0:1786176735.737147:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176735.737162:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176735.737171:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176735.737178:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786176735.737188:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176735.737197:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786176735.737215:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176735.737238:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176735.737245:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301635277
00000100:00000010:0.0:1786176735.737254:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176735.737264:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176735.737273:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301635277
00000100:00000010:0.0:1786176735.737279:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1851341e0.
00000100:00000001:0.0:1786176735.737287:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176735.737293:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176735.737298:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176735.737301:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176735.737307:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176735.737331:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176735.737336:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1786176735.737343:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176735.737348:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176735.737352:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176735.737362:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856291200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176735.737378:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176735.737382:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176735.737388:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176735.737393:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176735.737400:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176735.737410:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856291200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176735.737421:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176735.737426:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176735.737429:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176735.737433:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176735.737438:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176735.737443:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176735.737450:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176735.737454:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786176735.737462:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c44c8.
00000100:00000200:0.0:1786176735.737468:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856291200, portal 10
00000100:00000001:0.0:1786176735.737471:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176735.737474:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000040:0.0:1786176735.737486:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856291200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176735.737506:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176735.737550:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176735.737556:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4220.
00000100:00000200:0.0:1786176735.737561:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856291200, offset 0
00000400:00000200:0.0:1786176735.737569:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176735.737576:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176735.737581:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176735.737584:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176735.737587:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176735.737590:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176735.737592:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176735.737596:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254446:254446:255:4294967295] 192.168.202.110@tcp LPNI seq info [254435:254435:7:4294967295]
00000400:00000200:0.0:1786176735.737600:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176735.737607:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176735.737612:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176735.737616:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45200.
00000800:00000200:0.0:1786176735.737620:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176735.737624:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786176735.737628:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176735.737632:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176735.737636:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176735.737638:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301631498
00000100:00000001:0.0:1786176735.737641:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301635277
00000100:00000001:0.0:1786176735.737643:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301631498
00000100:00000001:0.0:1786176735.737645:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301635277
00000100:00000001:0.0:1786176735.737647:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176735.737650:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176735.737653:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176735.737657:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856291328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176735.737664:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176735.737666:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176735.737669:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176735.737670:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176735.737672:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176735.737675:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856291328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176735.737681:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176735.737682:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176735.737684:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176735.737685:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176735.737689:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176735.737691:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176735.737694:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176735.737696:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c3c8.
00000400:00000010:0.0:1786176735.737701:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4880.
00000100:00000200:0.0:1786176735.737704:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856291328, portal 10
00000100:00000001:0.0:1786176735.737706:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176735.737707:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:0.0:1786176735.737713:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856291328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176735.737719:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176735.737722:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176735.737724:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4908.
00000100:00000200:0.0:1786176735.737727:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856291328, offset 0
00000400:00000200:0.0:1786176735.737731:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176735.737734:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176735.737738:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176735.737741:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176735.737751:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176735.737753:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176735.737756:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176735.737759:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254447:254447:254:4294967295] 192.168.202.110@tcp LPNI seq info [254436:254436:6:4294967295]
00000400:00000200:0.0:1786176735.737763:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176735.737770:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176735.737774:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176735.737777:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45a00.
00000800:00000200:0.0:1786176735.737782:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176735.737786:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786176735.737790:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176735.737793:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176735.737796:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176735.737798:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301631623
00000100:00000001:0.0:1786176735.737800:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301635277
00000100:00000001:0.0:1786176735.737803:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301631623
00000100:00000001:0.0:1786176735.737805:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301635277
00000100:00000001:0.0:1786176735.737807:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176735.737809:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176735.737812:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176735.737816:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856291456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176735.737824:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176735.737825:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176735.737828:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176735.737830:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176735.737832:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176735.737835:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856291456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176735.737838:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176735.737840:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176735.737842:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176735.737844:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176735.737847:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176735.737849:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176735.737854:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176735.737857:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381cd68.
00000400:00000010:0.0:1786176735.737861:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176735.737864:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856291456, portal 4
00000100:00000001:0.0:1786176735.737866:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176735.737867:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:0.0:1786176735.737875:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856291456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176735.737882:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176735.737884:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176735.737887:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4990.
00000100:00000200:0.0:1786176735.737889:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856291456, offset 0
00000400:00000200:0.0:1786176735.737893:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176735.737898:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176735.737901:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176735.737904:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176735.737906:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176735.737908:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176735.737922:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176735.737928:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254448:254448:253:4294967295] 192.168.202.110@tcp LPNI seq info [254437:254437:5:4294967295]
00000400:00000200:0.0:1786176735.737933:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176735.737939:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176735.737943:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176735.737947:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786176735.737951:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176735.737955:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786176735.737958:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176735.737961:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176735.737964:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176735.737966:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301631498
00000100:00000001:0.0:1786176735.737968:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301635277
00000100:00000001:0.0:1786176735.737970:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301631498
00000100:00000001:0.0:1786176735.737972:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301635277
00000100:00000001:0.0:1786176735.737974:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176735.737976:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176735.737979:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176735.737981:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176735.737984:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176735.737987:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1786176735.737989:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176735.737990:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176735.737992:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176735.737996:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856291584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176735.738003:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176735.738005:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176735.738007:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176735.738009:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176735.738011:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176735.738015:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856291584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176735.738019:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176735.738021:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176735.738023:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176735.738025:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176735.738028:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176735.738030:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176735.738033:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176735.738036:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176735.738040:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176735.738042:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856291584, portal 4
00000100:00000001:0.0:1786176735.738044:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176735.738046:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:0.0:1786176735.738052:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856291584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176735.738059:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176735.738062:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176735.738064:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4bb0.
00000100:00000200:0.0:1786176735.738067:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856291584, offset 0
00000400:00000200:0.0:1786176735.738072:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176735.738076:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176735.738080:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176735.738083:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176735.738086:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176735.738088:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176735.738091:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176735.738095:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254449:254449:252:4294967295] 192.168.202.110@tcp LPNI seq info [254438:254438:4:4294967295]
00000400:00000200:0.0:1786176735.738098:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176735.738104:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176735.738108:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176735.738112:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176735.738116:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176735.738120:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786176735.738123:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176735.738126:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176735.738128:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176735.738130:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301631506
00000100:00000001:0.0:1786176735.738133:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301635277
00000100:00000001:0.0:1786176735.738135:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301631506
00000100:00000001:0.0:1786176735.738137:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301635277
00000100:00000001:0.0:1786176735.738138:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176735.738141:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176735.738143:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176735.738146:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176735.738148:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176735.738150:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1786176735.738152:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176735.738154:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176735.738155:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176735.738157:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176736.314000:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176736.314005:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176736.314008:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176737.336730:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176737.336785:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176737.336788:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:2.0:1786176738.808948:0:2379:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:2.0:1786176738.808991:0:2379:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1786176738.809002:0:2379:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176738.809006:0:2379:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:2.0:1786176738.809011:0:2379:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176738.809016:0:2379:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1786176738.809021:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176738.809024:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:2.0:1786176738.809030:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176738.809035:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176738.809042:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4f68
00000400:00000010:2.0:1786176738.809044:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4f68.
00000100:00000001:2.0:1786176738.809049:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176738.809058:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba69f800 x1872934856291072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176738.809085:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176738.809097:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856291072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176738.809106:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176738.809108:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176738.809119:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176738.809122:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45200.
00000400:00000200:2.0:1786176738.809125:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176738.809129:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176738.809133:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4220
00000400:00000010:2.0:1786176738.809135:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4220.
00000100:00000001:2.0:1786176738.809138:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1786176738.809141:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000200:2.0:1786176738.809144:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba69fb80 x1872934856291200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.809148:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176738.809155:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1786176738.809157:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176738.809166:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856291200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176738.809169:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176735/real 1786176738]  req@ffff97d1ba69f800 x1872934856291072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176738.809181:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176738.809186:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176738.809211:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176738.809214:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45a00.
00000400:00000200:2.0:1786176738.809217:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176738.809222:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000001:0.0:1786176738.809225:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176738.809228:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176738.809231:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786176738.809235:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786176738.809237:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786176738.809240:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000400:00000200:2.0:1786176738.809241:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4908
00000100:00000200:0.0:1786176738.809245:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856291072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1786176738.809246:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4908.
00000100:00000001:2.0:1786176738.809253:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176738.809262:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba69ca80 x1872934856291328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176738.809265:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856291072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.809275:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176738.809279:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.809284:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176738.809289:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176738.809289:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00100000:0.0:1786176738.809295:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856291072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.809303:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176738.809309:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856291072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.809318:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176738.809322:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.809324:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176738.809325:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.809329:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176738.809332:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176738.809336:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176738.809341:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176738.809345:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786176738.809355:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856291328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176738.809356:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856291072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176738.809386:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176738.809390:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856291072:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786176738.809391:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176738.809400:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:2.0:1786176738.809404:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000100:00000001:0.0:1786176738.809406:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000800:00000010:2.0:1786176738.809410:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000100:00000040:0.0:1786176738.809414:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856291072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786176738.809417:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176738.809427:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176738.809434:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4990
00000100:00000001:0.0:1786176738.809436:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000400:00000010:2.0:1786176738.809439:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4990.
02000000:00000001:0.0:1786176738.809441:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:2.0:1786176738.809444:0:2379:0:(events.c:34:request_out_callback()) Process entered
02000000:00000010:0.0:1786176738.809448:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
00000100:00000200:2.0:1786176738.809453:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba69f480 x1872934856291456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786176738.809456:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176738.809460:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176738.809464:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176738.809469:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00000001:2.0:1786176738.809470:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000010:0.0:1786176738.809473:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
00000100:00000040:2.0:1786176738.809477:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856291456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786176738.809480:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176738.809483:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176738.809487:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000001:2.0:1786176738.809492:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176738.809494:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786176738.809496:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176738.809497:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000800:00000100:2.0:1786176738.809505:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176738.809509:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000100:00000001:0.0:1786176738.809541:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176738.809545:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176738.809547:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176738.809549:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176738.809556:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176735/real 1786176738]  req@ffff97d1ba69fb80 x1872934856291200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786176738.809569:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000001:0.0:1786176738.809570:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176738.809573:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176738.809577:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:2.0:1786176738.809579:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1786176738.809583:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c44c8
00000400:00000010:0.0:1786176738.809587:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c44c8.
00000100:00000001:0.0:1786176738.809591:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000400:00000200:2.0:1786176738.809592:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4bb0
00000100:00000200:0.0:1786176738.809597:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856291200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1786176738.809598:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4bb0.
00000100:00000001:2.0:1786176738.809605:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176738.809614:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba69f100 x1872934856291584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176738.809617:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856291200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.809634:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176738.809636:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.809642:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176738.809644:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176738.809645:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00100000:0.0:1786176738.809650:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856291200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176738.809653:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856291584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176738.809671:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176738.809673:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176738.809685:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176738.809692:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856291200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.809700:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176738.809703:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.809704:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176738.809705:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.809707:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176738.809710:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176738.809712:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176738.809716:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176738.809719:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176738.809722:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856291200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176738.809730:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856291200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176738.809734:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176738.809739:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176735/real 1786176738]  req@ffff97d1ba69ca80 x1872934856291328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176738.809747:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176738.809748:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176738.809750:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c3c8.
00000400:00000200:0.0:1786176738.809753:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4880
00000400:00000010:0.0:1786176738.809755:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4880.
00000100:00000001:0.0:1786176738.809757:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176738.809760:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856291328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176738.809768:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856291328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.809774:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176738.809776:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.809777:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176738.809779:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176738.809783:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856291328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.809790:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176738.809794:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856291328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.809802:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176738.809805:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.809807:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176738.809808:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.809810:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176738.809812:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176738.809814:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176738.809817:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176738.809820:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176738.809824:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856291328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176738.809833:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856291328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176738.809853:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176738.809858:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176735/real 1786176738]  req@ffff97d1ba69f480 x1872934856291456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176738.809867:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176738.809869:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176738.809871:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381cd68.
00000400:00000200:0.0:1786176738.809873:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176738.809875:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176738.809878:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176738.809882:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856291456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176738.809892:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856291456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.809899:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176738.809901:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.809903:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176738.809905:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176738.809909:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856291456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.809917:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176738.809921:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856291456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.809930:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176738.809933:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.809935:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176738.809936:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.809938:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176738.809942:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176738.809945:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176738.809949:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176738.809953:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176738.810434:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856291456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176738.810449:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856291456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176738.810454:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176738.810457:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176738.810461:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856291200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.810469:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176738.810471:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176738.810474:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176738.810477:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176738.810479:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176738.810481:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176738.810484:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176738.810487:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786176738.810490:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176738.810492:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176738.810494:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:0.0:1786176738.810497:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176738.810498:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.810500:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176738.810504:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856291328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.810552:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176738.810554:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176738.810556:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176738.810560:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176738.810562:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176738.810564:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176738.810566:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176738.810568:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786176738.810571:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176738.810573:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176738.810574:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:0.0:1786176738.810577:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176738.810578:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.810580:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176738.810584:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856291456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.810591:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176738.810592:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176738.810594:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176738.810597:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176738.810598:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176738.810599:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176738.810601:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176738.810603:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786176738.810606:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176738.810607:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176738.810609:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:0.0:1786176738.810611:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176738.810612:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.810615:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176738.810617:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176738.810619:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176738.810621:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176738.810625:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176735/real 1786176738]  req@ffff97d1ba69f100 x1872934856291584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176738.810642:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176738.810643:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176738.810645:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176738.810648:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176738.810650:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176738.810652:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176738.810656:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856291584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176738.810664:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856291584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.810671:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176738.810673:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.810675:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176738.810677:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176738.810681:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856291584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.810688:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176738.810693:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856291584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.810700:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176738.810703:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.810704:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176738.810706:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.810708:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176738.810710:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176738.810713:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176738.810717:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176738.810720:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176738.810725:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856291584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176738.810733:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856291584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176738.810737:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.810739:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176738.810742:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856291584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176738.810749:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176738.810751:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176738.810752:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176738.810755:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176738.810757:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176738.810758:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176738.810760:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176738.810761:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786176738.810763:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176738.810765:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176738.810766:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:0.0:1786176738.810769:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176738.810770:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.810772:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176738.810779:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176738.810781:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176738.810783:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176738.810784:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176739.384782:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176739.384787:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176739.384790:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176739.576599:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301635277, at 4301635328
00000100:00000010:0.0:1786176739.576614:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000020:00000001:0.0:1786176740.408745:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176740.408748:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176740.408751:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786176740.856914:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176740.856919:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176740.856923:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176740.856934:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856291712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.856955:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176740.856959:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176740.856966:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176740.856971:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176740.856977:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176740.856986:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856291712:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176740.856995:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176740.857002:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176740.857007:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176740.857014:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176740.857023:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176740.857029:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176740.857039:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176740.857046:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176740.857057:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176740.857070:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856291712, portal 25
00000100:00000001:0.0:1786176740.857076:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176740.857080:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000100:00000040:0.0:1786176740.857091:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856291712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.857108:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176740.857115:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176740.857121:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176740.857128:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856291712, offset 0
00000400:00000200:0.0:1786176740.857143:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176740.857157:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176740.857187:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176740.857192:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176740.857197:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176740.857200:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176740.857203:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176740.857210:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254450:254450:256:4294967295] 192.168.202.110@tcp LPNI seq info [254439:254439:8:4294967295]
00000400:00000200:0.0:1786176740.857221:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176740.857243:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176740.857256:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176740.857266:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176740.857279:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176740.857287:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176740.857300:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176740.857306:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1786176740.857317:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176740.857323:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176740.857331:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176740.857341:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176740.857351:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176740.857357:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176740.857365:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176740.857375:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856291712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.857401:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176740.857411:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856291712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.857429:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.857432:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176740.857445:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176740.857454:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176740.857461:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301640397
00000100:00000010:0.0:1786176740.857471:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176740.857484:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176740.857494:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301640397
00000100:00000010:0.0:1786176740.857503:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134480.
00000100:00000001:0.0:1786176740.857508:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176740.857554:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176740.857558:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.857560:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176740.857568:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176740/real 1786176740]  req@ffff97d1ba69d500 x1872934856291712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176740.857581:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176740.857583:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176740.857586:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176740.857589:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176740.857590:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176740.857594:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176740.857597:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856291712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176740.857605:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856291712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.857612:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176740.857614:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.857616:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176740.857619:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176740.857623:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856291712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.857629:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176740.857634:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856291712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.857641:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176740.857644:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.857646:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176740.857647:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.857650:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176740.857653:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176740.857656:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176740.857660:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176740.857665:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176740.857670:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856291712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176740.857684:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856291712:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176740.857695:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.857701:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176740.857710:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856291712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.857728:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176740.857732:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176740.857738:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176740.857744:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176740.857748:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176740.857750:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176740.857754:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176740.857759:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786176740.857765:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176740.857769:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176740.857773:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:0.0:1786176740.857779:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176740.857783:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.857789:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.857862:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176740.857867:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.857874:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176740.857879:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176740.857883:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176740.857894:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856291840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.857915:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176740.857920:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176740.857927:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176740.857932:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176740.857938:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176740.857946:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856291840:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176740.857962:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176740.857967:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176740.857971:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176740.857976:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176740.857982:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176740.857987:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176740.857993:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176740.858000:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176740.858007:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176740.858016:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856291840, portal 10
00000100:00000001:0.0:1786176740.858021:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176740.858025:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:0.0:1786176740.858038:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856291840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858061:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176740.858068:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176740.858075:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176740.858083:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856291840, offset 0
00000400:00000200:0.0:1786176740.858095:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176740.858104:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176740.858112:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176740.858120:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176740.858124:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176740.858127:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176740.858129:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176740.858133:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254451:254451:256:4294967295] 192.168.202.110@tcp LPNI seq info [254440:254440:8:4294967295]
00000400:00000200:0.0:1786176740.858138:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176740.858145:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176740.858150:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176740.858154:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176740.858159:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176740.858162:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176740.858166:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176740.858169:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1786176740.858191:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176740.858193:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176740.858196:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176740.858200:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176740.858204:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176740.858205:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176740.858208:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176740.858212:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856291840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858220:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176740.858223:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856291840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858229:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.858231:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176740.858234:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176740.858239:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176740.858242:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176740.858243:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301636618
00000100:00000001:0.0:1786176740.858246:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301640397
00000100:00000001:0.0:1786176740.858248:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301636618
00000100:00000001:0.0:1786176740.858251:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301640397
00000100:00000001:0.0:1786176740.858253:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176740.858255:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.858256:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176740.858261:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176740/real 1786176740]  req@ffff97d1ba69ea00 x1872934856291840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176740.858275:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176740.858277:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176740.858279:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176740.858281:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176740.858283:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176740.858284:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176740.858287:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856291840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176740.858295:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856291840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858302:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176740.858303:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.858305:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176740.858306:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176740.858311:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856291840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858318:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176740.858323:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856291840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858330:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176740.858332:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.858334:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176740.858335:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.858337:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176740.858339:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176740.858341:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176740.858345:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176740.858348:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176740.858353:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856291840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176740.858362:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856291840:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176740.858367:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176740.858370:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856291968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858376:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176740.858378:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176740.858381:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176740.858383:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176740.858385:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176740.858389:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856291968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176740.858392:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176740.858395:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176740.858396:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176740.858399:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176740.858403:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176740.858405:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176740.858408:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176740.858411:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176740.858415:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176740.858418:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856291968, portal 10
00000100:00000001:0.0:1786176740.858420:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176740.858421:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:0.0:1786176740.858428:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856291968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858435:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176740.858437:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176740.858440:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176740.858443:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856291968, offset 0
00000400:00000200:0.0:1786176740.858448:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176740.858454:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176740.858459:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176740.858462:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176740.858466:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176740.858468:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176740.858470:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176740.858473:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254452:254452:256:4294967295] 192.168.202.110@tcp LPNI seq info [254441:254441:8:4294967295]
00000400:00000200:0.0:1786176740.858477:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176740.858482:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176740.858485:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176740.858488:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176740.858491:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176740.858494:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176740.858498:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176740.858501:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1786176740.858506:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176740.858508:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176740.858537:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176740.858541:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176740.858545:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176740.858547:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176740.858549:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176740.858553:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856291968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858562:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176740.858566:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856291968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858574:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.858576:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176740.858580:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176740.858583:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176740.858587:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176740.858590:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301636744
00000100:00000001:0.0:1786176740.858593:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301640397
00000100:00000001:0.0:1786176740.858595:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301636744
00000100:00000001:0.0:1786176740.858597:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301640397
00000100:00000001:0.0:1786176740.858599:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176740.858602:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.858603:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176740.858608:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176740/real 1786176740]  req@ffff97d1ba69c000 x1872934856291968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176740.858616:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176740.858618:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176740.858620:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176740.858622:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176740.858624:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176740.858626:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176740.858629:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856291968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176740.858638:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856291968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858646:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176740.858649:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.858651:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176740.858653:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176740.858657:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856291968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858666:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176740.858672:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856291968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858680:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176740.858683:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.858685:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176740.858686:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.858688:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176740.858690:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176740.858694:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176740.858698:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176740.858701:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176740.858706:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856291968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176740.858715:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856291968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176740.858720:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176740.858724:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856292096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858729:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176740.858730:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176740.858732:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176740.858733:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176740.858735:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176740.858737:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856292096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176740.858738:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176740.858740:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176740.858741:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176740.858742:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176740.858745:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176740.858747:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176740.858750:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176740.858752:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176740.858756:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176740.858760:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856292096, portal 4
00000100:00000001:0.0:1786176740.858762:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176740.858764:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:0.0:1786176740.858770:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856292096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858777:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176740.858779:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176740.858782:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176740.858784:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856292096, offset 0
00000400:00000200:0.0:1786176740.858788:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176740.858793:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176740.858797:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176740.858800:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176740.858803:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176740.858806:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176740.858807:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176740.858810:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254453:254453:256:4294967295] 192.168.202.110@tcp LPNI seq info [254442:254442:8:4294967295]
00000400:00000200:0.0:1786176740.858813:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176740.858818:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176740.858821:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176740.858823:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176740.858826:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176740.858828:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176740.858831:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176740.858833:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1786176740.858835:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176740.858837:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176740.858839:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176740.858841:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176740.858844:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176740.858845:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176740.858847:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176740.858849:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856292096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858859:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176740.858862:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856292096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858870:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.858872:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176740.858876:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176740.858879:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176740.858883:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176740.858886:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301636619
00000100:00000001:0.0:1786176740.858888:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301640397
00000100:00000001:0.0:1786176740.858890:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301636619
00000100:00000001:0.0:1786176740.858893:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301640397
00000100:00000001:0.0:1786176740.858895:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176740.858897:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.858899:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176740.858904:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176740/real 1786176740]  req@ffff97d1ba69c700 x1872934856292096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176740.858913:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176740.858914:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176740.858916:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176740.858919:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176740.858920:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176740.858922:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176740.858925:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856292096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176740.858930:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856292096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858935:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176740.858936:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.858938:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176740.858939:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176740.858941:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856292096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858945:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176740.858948:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856292096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858952:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176740.858953:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.858954:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176740.858955:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.858957:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176740.858958:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176740.858959:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176740.858961:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176740.858963:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176740.858965:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856292096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176740.858970:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856292096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176740.858972:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.858973:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176740.858975:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856291840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.858979:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176740.858981:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176740.858982:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176740.858984:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176740.858985:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176740.858987:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176740.858988:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176740.858990:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786176740.858992:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176740.858993:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176740.858994:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:0.0:1786176740.858996:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176740.858997:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.858999:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176740.859001:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856291968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.859005:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176740.859005:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176740.859006:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176740.859008:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176740.859008:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176740.859009:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176740.859010:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176740.859011:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786176740.859012:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176740.859013:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176740.859013:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:0.0:1786176740.859015:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176740.859016:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.859016:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176740.859018:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856292096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.859022:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176740.859023:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176740.859023:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176740.859025:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176740.859025:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176740.859026:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176740.859027:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176740.859028:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786176740.859030:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176740.859031:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176740.859032:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:0.0:1786176740.859033:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176740.859034:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.859036:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.859039:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176740.859040:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.859041:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176740.859042:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176740.859044:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176740.859046:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856292224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.859051:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176740.859052:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176740.859053:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176740.859054:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176740.859056:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176740.859057:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856292224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176740.859059:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176740.859061:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176740.859062:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176740.859063:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176740.859065:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176740.859067:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176740.859070:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176740.859073:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176740.859077:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176740.859080:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856292224, portal 4
00000100:00000001:0.0:1786176740.859082:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176740.859083:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:0.0:1786176740.859090:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856292224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.859096:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176740.859098:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176740.859101:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176740.859104:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856292224, offset 0
00000400:00000200:0.0:1786176740.859108:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176740.859112:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176740.859115:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176740.859119:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176740.859121:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176740.859124:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176740.859126:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176740.859130:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254454:254454:256:4294967295] 192.168.202.110@tcp LPNI seq info [254443:254443:8:4294967295]
00000400:00000200:0.0:1786176740.859134:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176740.859141:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176740.859146:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176740.859149:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176740.859152:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176740.859153:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786176740.859156:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176740.859159:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1786176740.859161:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176740.859163:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176740.859165:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176740.859167:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176740.859169:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176740.859185:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176740.859188:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176740.859192:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856292224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.859197:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176740.859199:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856292224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.859204:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.859205:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176740.859208:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176740.859211:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176740.859213:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176740.859215:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301636627
00000100:00000001:0.0:1786176740.859216:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301640397
00000100:00000001:0.0:1786176740.859218:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301636627
00000100:00000001:0.0:1786176740.859220:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301640397
00000100:00000001:0.0:1786176740.859221:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176740.859222:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.859223:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176740.859226:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176740/real 1786176740]  req@ffff97d1ba69e300 x1872934856292224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176740.859231:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176740.859232:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176740.859233:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176740.859234:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176740.859236:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176740.859237:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176740.859239:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856292224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176740.859244:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856292224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.859248:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176740.859249:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.859250:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176740.859251:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176740.859253:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856292224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.859257:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176740.859259:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856292224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.859265:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176740.859266:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.859267:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176740.859268:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.859269:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176740.859271:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176740.859272:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176740.859274:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176740.859275:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176740.859278:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856292224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176740.859282:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856292224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176740.859285:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.859286:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176740.859288:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856292224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176740.859291:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176740.859292:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176740.859294:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176740.859295:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176740.859296:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176740.859297:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176740.859298:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176740.859299:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786176740.859301:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176740.859302:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176740.859303:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:0.0:1786176740.859305:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176740.859306:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.859307:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176740.859309:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176740.859310:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176740.859311:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176740.859313:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176742.456751:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176742.456758:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176742.456764:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176743.480687:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176743.480691:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176743.480694:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176744.696602:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301640397, at 4301640448
00000100:00000010:0.0:1786176744.696618:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000020:00000001:2.0:1786176745.528801:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176745.528808:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176745.528811:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176745.976694:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176745.976707:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176745.976710:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176745.976715:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176745.976721:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176745.976731:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6969
00000400:00000200:2.0:1786176745.976742:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176745.976751:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176745.976756:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176745.976760:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.976764:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176745.976767:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.976770:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176745.976775:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176745.976778:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176745.976781:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176745.976785:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176745.976788:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176745.976800:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176745.976804:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.976807:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176745.976809:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.976812:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176745.976816:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176745.976819:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176745.976822:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176745.976826:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176745.976830:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176745.976834:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.976837:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176745.976839:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176745.976852:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786176745.976857:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176745.976862:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176745.976864:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.976868:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176745.976871:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176745.976878:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176745.976889:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786176745.976897:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176745.976911:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176745.976924:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176745.976930:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176745.976936:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176745.976963:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176745.976967:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176745.976971:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176745.976973:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176745.976975:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176745.976978:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176745.976981:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6969
00000400:00000200:2.0:1786176745.976987:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176745.976991:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176745.976994:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176745.976998:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176745.977000:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786176745.977002:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786176745.977007:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1786176745.977007:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176745.977011:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786176745.977013:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176745.977019:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000040:0.0:1786176745.977020:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176745.977022:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176745.977026:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176745.977030:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:0.0:1786176745.977035:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000001:2.0:1786176745.977036:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786176745.977038:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176745.977041:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:0.0:1786176745.977044:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1786176745.977049:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:0.0:1786176745.977050:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786176745.977054:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1786176745.977056:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176745.977060:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00100000:0.0:1786176745.977066:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856292352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786176745.977066:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176745.977071:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:0.0:1786176745.977076:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:2.0:1786176745.977078:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176745.977082:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176745.977083:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786176745.977084:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176745.977089:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176745.977094:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1786176745.977094:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176745.977100:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176745.977102:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
00000100:00000001:2.0:1786176745.977103:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:0.0:1786176745.977107:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1786176745.977111:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1786176745.977116:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:2.0:1786176745.977119:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.977137:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176745.977142:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000400:00000010:0.0:1786176745.977144:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000020:00000040:2.0:1786176745.977145:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176745.977151:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176745.977153:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786176745.977156:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000001:2.0:1786176745.977156:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176745.977160:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176745.977165:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786176745.977167:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856292352, portal 25
00000100:00000001:0.0:1786176745.977171:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
02000000:00000010:2.0:1786176745.977171:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:0.0:1786176745.977173:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000001:2.0:1786176745.977179:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176745.977182:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786176745.977190:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176745.977193:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176745.977201:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176745.977206:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000010:0.0:1786176745.977208:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000001:2.0:1786176745.977212:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786176745.977215:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856292352, offset 0
00000100:00000040:2.0:1786176745.977221:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786176745.977230:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176745.977240:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176745.977243:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1786176745.977247:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786176745.977251:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176745.977255:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1786176745.977258:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:2.0:1786176745.977259:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176745.977264:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1786176745.977266:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1786176745.977270:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6969
00000400:00000200:0.0:1786176745.977272:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176745.977279:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786176745.977281:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176745.977284:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786176745.977290:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:0.0:1786176745.977291:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254455:254455:256:4294967295] 192.168.202.110@tcp LPNI seq info [254444:254444:8:4294967295]
00000100:00080000:2.0:1786176745.977297:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786176745.977299:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786176745.977306:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.977312:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786176745.977316:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1786176745.977319:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.977324:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:0.0:1786176745.977327:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000001:2.0:1786176745.977329:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176745.977332:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000800:00000010:0.0:1786176745.977334:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000020:00000001:2.0:1786176745.977335:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176745.977339:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176745.977344:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:0.0:1786176745.977345:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176745.977350:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:2.0:1786176745.977351:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176745.977361:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:0.1:1786176745.977361:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176745.977367:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1786176745.977370:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:2.0:1786176745.977371:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.977379:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176745.977381:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000800:00000100:0.0:1786176745.977385:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176745.977388:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000010:0.0:1786176745.977391:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000100:00000001:2.0:1786176745.977393:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176745.977397:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176745.977401:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1786176745.977402:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1786176745.977406:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.977410:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:0.0:1786176745.977412:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000008:00100000:2.0:1786176745.977415:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1786176745.977418:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000008:00000001:2.0:1786176745.977420:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786176745.977422:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:2.0:1786176745.977425:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.977428:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000010:2.0:1786176745.977431:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000100:00000200:0.0:1786176745.977434:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176745.977436:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176745.977443:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176745.977446:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.977449:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786176745.977450:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176745.977454:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:0.0:1786176745.977455:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176745.977459:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176745.977467:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786176745.977473:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.977478:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176745.977482:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176745.977483:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176745.977493:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.977494:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176745.977499:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176745.977501:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786176745.977506:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.977548:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301645517
00000100:00000001:2.0:1786176745.977549:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176745.977556:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:0.0:1786176745.977560:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00080000:2.0:1786176745.977561:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176745.977566:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176745.977568:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176745.977572:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176745.977582:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301645518
00000100:00000010:0.0:1786176745.977587:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134c60.
00000100:00000001:0.0:1786176745.977590:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176745.977593:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176745.977595:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.977597:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176745.977602:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176745/real 1786176745]  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176745.977610:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176745.977611:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176745.977613:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176745.977615:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176745.977617:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176745.977619:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176745.977622:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176745.977632:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.977640:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176745.977642:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.977645:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176745.977648:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176745.977653:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.977663:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176745.977668:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.977675:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176745.977678:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.977679:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176745.977680:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.977683:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176745.977694:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176745.977697:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176745.977700:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176745.977702:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176745.977706:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176745.977712:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856292352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176745.977716:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.977718:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176745.977719:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856292352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.977724:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176745.977727:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176745.977729:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176745.977732:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176745.977734:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176745.977735:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176745.977737:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176745.977740:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:0.0:1786176745.977743:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176745.977746:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176745.977747:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:0.0:1786176745.977750:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176745.977751:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.977753:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.977761:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176745.977762:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.977764:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176745.977765:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176745.977767:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176745.977770:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.977778:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176745.977780:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176745.977782:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176745.977784:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176745.977787:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176745.977790:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856292480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176745.977794:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176745.977796:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176745.977798:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176745.977800:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176745.977803:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176745.977806:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176745.977810:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176745.977814:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176745.977818:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176745.977822:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856292480, portal 10
00000100:00000001:0.0:1786176745.977823:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176745.977824:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:0.0:1786176745.977832:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.977839:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176745.977842:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176745.977845:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176745.977848:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856292480, offset 0
00000400:00000200:0.0:1786176745.977852:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176745.977857:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176745.977861:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176745.977864:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176745.977867:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176745.977870:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176745.977872:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176745.977876:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254456:254456:256:4294967295] 192.168.202.110@tcp LPNI seq info [254445:254445:8:4294967295]
00000400:00000200:0.0:1786176745.977880:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176745.977887:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176745.977892:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176745.977896:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176745.977900:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176745.977903:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176745.977908:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176745.977911:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786176745.977916:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176745.977918:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176745.977921:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176745.977925:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176745.977928:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176745.977930:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176745.977932:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176745.977934:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.977943:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176745.977946:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.977953:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.977955:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176745.977960:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176745.977965:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176745.977969:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176745.977971:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301641738
00000100:00000001:0.0:1786176745.977974:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301645517
00000100:00000001:0.0:1786176745.977977:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301641738
00000100:00000001:0.0:1786176745.977979:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301645518
00000100:00000001:0.0:1786176745.977997:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176745.978000:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.978002:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176745.978007:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176745/real 1786176745]  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176745.978014:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176745.978015:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176745.978017:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176745.978019:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176745.978020:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176745.978022:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176745.978024:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176745.978030:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.978034:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176745.978035:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.978036:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176745.978037:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176745.978040:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.978047:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176745.978051:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.978060:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176745.978063:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.978064:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176745.978066:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.978068:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176745.978070:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176745.978073:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176745.978077:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176745.978095:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176745.978100:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176745.978110:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856292480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176745.978113:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176745.978115:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.978120:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176745.978137:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176745.978139:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176745.978140:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176745.978142:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176745.978144:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856292608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176745.978147:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176745.978148:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176745.978150:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176745.978151:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176745.978154:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176745.978155:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176745.978158:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176745.978160:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176745.978163:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176745.978165:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856292608, portal 10
00000100:00000001:0.0:1786176745.978167:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176745.978168:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:0.0:1786176745.978172:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.978177:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176745.978179:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176745.978180:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176745.978182:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856292608, offset 0
00000400:00000200:0.0:1786176745.978185:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176745.978187:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176745.978190:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176745.978192:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176745.978195:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176745.978197:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176745.978200:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176745.978203:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254457:254457:256:4294967295] 192.168.202.110@tcp LPNI seq info [254446:254446:8:4294967295]
00000400:00000200:0.0:1786176745.978207:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176745.978214:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176745.978219:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176745.978223:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176745.978227:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176745.978230:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176745.978235:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176745.978238:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786176745.978242:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176745.978245:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176745.978247:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176745.978250:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176745.978252:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176745.978254:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176745.978261:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176745.978264:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.978270:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176745.978272:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.978277:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.978278:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176745.978280:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176745.978283:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176745.978285:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176745.978287:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301641863
00000100:00000001:0.0:1786176745.978289:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301645517
00000100:00000001:0.0:1786176745.978290:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301641863
00000100:00000001:0.0:1786176745.978292:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301645518
00000100:00000001:0.0:1786176745.978293:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176745.978295:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.978296:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176745.978298:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176745/real 1786176745]  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176745.978303:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176745.978304:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176745.978306:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176745.978308:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176745.978309:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176745.978310:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176745.978312:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176745.978318:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.978323:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176745.978324:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.978325:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176745.978326:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176745.978329:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.978333:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176745.978335:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.978339:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176745.978340:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.978341:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176745.978342:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.978343:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176745.978345:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176745.978346:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176745.978348:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176745.978349:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176745.978352:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176745.978356:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856292608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176745.978361:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.978363:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176745.978366:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856292480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.978373:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176745.978376:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176745.978378:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176745.978381:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176745.978382:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176745.978384:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176745.978386:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176745.978388:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:0.0:1786176745.978390:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176745.978392:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176745.978394:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:0.0:1786176745.978397:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176745.978398:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.978400:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176745.978404:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856292608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.978411:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176745.978412:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176745.978413:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176745.978415:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176745.978416:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176745.978417:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176745.978418:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176745.978419:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:0.0:1786176745.978421:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176745.978421:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176745.978422:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:0.0:1786176745.978424:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176745.978425:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.978426:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.978428:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176745.978429:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.978430:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176745.978432:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.978939:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176745.978945:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6969
00000400:00000200:2.0:1786176745.978953:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176745.978958:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176745.978960:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176745.978963:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.978966:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176745.978968:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.978970:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176745.978973:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176745.978974:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176745.978976:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176745.978979:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176745.978980:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176745.978982:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176745.978984:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.978985:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176745.978987:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.978999:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176745.979001:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176745.979003:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176745.979005:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176745.979007:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176745.979013:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176745.979015:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.979017:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176745.979021:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176745.979023:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.979024:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176745.979033:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176745.979035:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176745.979038:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176745.979039:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.979042:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176745.979044:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176745.979047:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176745.979053:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786176745.979058:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176745.979066:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176745.979074:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176745.979077:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176745.979081:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176745.979108:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176745.979111:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176745.979115:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176745.979116:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176745.979118:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176745.979131:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176745.979134:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6969
00000400:00000200:2.0:1786176745.979137:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176745.979141:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176745.979143:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176745.979146:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.979149:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176745.979151:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.979153:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176745.979155:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176745.979156:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176745.979157:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786176745.979160:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1786176745.979160:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176745.979165:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:0.0:1786176745.979166:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1786176745.979168:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:0.0:1786176745.979172:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786176745.979173:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.979176:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176745.979178:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176745.979184:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:0.0:1786176745.979185:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176745.979187:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786176745.979195:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176745.979198:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176745.979200:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176745.979202:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176745.979206:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.979210:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176745.979212:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176745.979213:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.979220:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176745.979226:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786176745.979233:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:0.0:1786176745.979240:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1786176745.979240:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176745.979246:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.979247:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.979249:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176745.979254:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176745.979256:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176745.979257:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:2.0:1786176745.979260:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
02000000:00000001:0.0:1786176745.979263:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786176745.979266:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176745.979269:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176745.979278:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856292736:192.168.202.110@tcp:8:kworker.0
00000100:00100000:2.0:1786176745.979278:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176745.979294:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176745.979299:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176745.979303:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176745.979308:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176745.979314:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176745.979321:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176745.979326:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176745.979329:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176745.979337:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786176745.979343:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1786176745.979346:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176745.979358:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000001:2.0:1786176745.979361:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786176745.979366:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856292736, portal 4
00000100:00000001:0.0:1786176745.979371:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000040:2.0:1786176745.979371:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6979)
00000100:00000001:0.0:1786176745.979376:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:0.0:1786176745.979388:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979396:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176745.979399:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176745.979402:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176745.979404:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856292736, offset 0
00000400:00000200:0.0:1786176745.979409:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176745.979414:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176745.979419:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176745.979422:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176745.979425:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176745.979427:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176745.979430:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176745.979434:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254458:254458:256:4294967295] 192.168.202.110@tcp LPNI seq info [254447:254447:8:4294967295]
00000400:00000200:0.0:1786176745.979439:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176745.979445:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176745.979450:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176745.979454:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176745.979458:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176745.979461:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176745.979467:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176745.979471:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786176745.979476:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176745.979478:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176745.979481:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176745.979485:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176745.979487:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176745.979489:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176745.979491:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176745.979493:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979500:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176745.979502:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979507:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.979508:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176745.979528:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176745.979531:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176745.979533:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176745.979536:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301641740
00000100:00000001:0.0:1786176745.979539:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301645517
00000100:00000001:0.0:1786176745.979541:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301641740
00000100:00000001:0.0:1786176745.979544:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301645518
00000100:00000001:0.0:1786176745.979546:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176745.979549:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.979550:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176745.979555:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176745/real 1786176745]  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176745.979565:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176745.979566:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176745.979568:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176745.979571:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176745.979573:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176745.979576:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176745.979580:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176745.979589:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979595:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176745.979596:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.979598:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176745.979599:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176745.979601:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979606:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176745.979609:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979613:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176745.979615:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.979616:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176745.979616:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.979618:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176745.979619:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176745.979621:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176745.979624:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176745.979627:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176745.979631:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176745.979640:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856292736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176745.979644:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.979646:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176745.979650:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856292736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979657:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176745.979659:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176745.979662:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176745.979665:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176745.979667:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176745.979668:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176745.979671:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176745.979672:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:0.0:1786176745.979674:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176745.979676:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176745.979677:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:0.0:1786176745.979679:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176745.979680:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.979681:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.979684:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176745.979685:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.979687:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176745.979688:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176745.979690:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176745.979693:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979698:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176745.979700:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176745.979702:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176745.979703:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176745.979705:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176745.979707:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856292864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176745.979709:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176745.979711:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176745.979712:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176745.979713:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176745.979716:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176745.979717:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176745.979720:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176745.979722:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176745.979724:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176745.979726:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856292864, portal 4
00000100:00000001:0.0:1786176745.979728:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176745.979729:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000040:0.0:1786176745.979734:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979738:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176745.979740:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176745.979741:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176745.979743:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856292864, offset 0
00000400:00000200:0.0:1786176745.979746:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176745.979748:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176745.979751:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176745.979753:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176745.979755:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176745.979757:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176745.979758:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176745.979761:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254459:254459:256:4294967295] 192.168.202.110@tcp LPNI seq info [254448:254448:8:4294967295]
00000400:00000200:0.0:1786176745.979765:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176745.979772:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176745.979777:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176745.979780:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176745.979785:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176745.979787:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786176745.979792:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176745.979796:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786176745.979800:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176745.979802:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176745.979806:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176745.979809:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176745.979813:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176745.979815:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176745.979817:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176745.979820:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979826:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176745.979828:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979832:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.979834:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176745.979836:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176745.979839:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176745.979841:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176745.979843:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301641748
00000100:00000001:0.0:1786176745.979845:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301645517
00000100:00000001:0.0:1786176745.979846:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301641748
00000100:00000001:0.0:1786176745.979848:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301645518
00000100:00000001:0.0:1786176745.979849:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176745.979850:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.979852:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176745.979854:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176745/real 1786176745]  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176745.979860:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176745.979861:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176745.979863:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176745.979864:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176745.979865:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176745.979866:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176745.979868:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176745.979874:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979878:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176745.979879:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.979880:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176745.979881:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176745.979883:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979887:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176745.979889:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979893:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176745.979895:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.979896:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176745.979897:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.979899:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176745.979900:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176745.979901:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176745.979903:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176745.979905:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176745.979908:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176745.979913:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856292864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176745.979916:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.979917:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176745.979919:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856292864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176745.979922:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176745.979923:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176745.979925:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176745.979926:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176745.979927:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176745.979928:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176745.979929:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176745.979930:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:0.0:1786176745.979931:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176745.979932:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176745.979933:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:0.0:1786176745.979936:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176745.979937:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.979939:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176745.979943:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176745.979944:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176745.979946:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176745.979948:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176746.552702:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176746.552707:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176746.552711:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176748.600682:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176748.600685:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176748.600688:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176749.624708:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176749.624711:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176749.624714:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176749.816612:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301645517, at 4301645568
00000100:00000010:0.0:1786176749.816629:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00080000:2.0:1786176751.096593:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176751.096601:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176751.096603:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176751.096606:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176751.096610:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176751.096617:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6974
00000400:00000200:2.0:1786176751.096626:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176751.096635:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176751.096638:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176751.096643:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.096646:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176751.096650:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.096654:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176751.096657:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176751.096660:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176751.096662:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176751.096666:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176751.096670:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176751.096671:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176751.096674:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.096676:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176751.096677:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.096679:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176751.096681:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176751.096684:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176751.096686:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176751.096688:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176751.096692:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176751.096694:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176751.096695:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176751.096696:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176751.096705:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000020:00000040:2.0:1786176751.096708:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176751.096711:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176751.096712:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176751.096715:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176751.096717:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176751.096721:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176751.096727:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786176751.096732:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176751.096739:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176751.096747:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176751.096750:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176751.096753:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176751.096771:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176751.096774:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176751.096777:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176751.096778:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176751.096779:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176751.096781:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176751.096783:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6974
00000400:00000200:2.0:1786176751.096786:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176751.096788:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176751.096790:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176751.096792:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.096793:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176751.096796:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.096797:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176751.096799:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176751.096800:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176751.096801:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176751.096802:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176751.096804:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176751.096805:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176751.096808:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176751.096809:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786176751.096810:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786176751.096815:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1786176751.096815:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176751.096819:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786176751.096819:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176751.096823:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176751.096827:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:0.0:1786176751.096829:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176751.096829:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176751.096833:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176751.096838:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176751.096840:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1786176751.096841:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.096844:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176751.096845:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:0.0:1786176751.096849:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:2.0:1786176751.096850:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1786176751.096853:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:2.0:1786176751.096854:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176751.096858:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176751.096859:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176751.096862:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000100:00100000:0.0:1786176751.096866:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856292992:192.168.202.110@tcp:250:kworker.0
00000020:00000040:2.0:1786176751.096867:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176751.096871:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1786176751.096874:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786176751.096874:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176751.096878:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176751.096879:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786176751.096880:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176751.096884:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176751.096884:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176751.096890:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786176751.096891:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000001:2.0:1786176751.096894:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176751.096896:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
00000100:00100000:2.0:1786176751.096900:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176751.096906:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:0.0:1786176751.096907:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176751.096908:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176751.096913:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786176751.096921:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176751.096923:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786176751.096925:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000100:00080000:2.0:1786176751.096927:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176751.096931:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176751.096932:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:0.0:1786176751.096934:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00080000:2.0:1786176751.096934:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176751.096937:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176751.096940:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6974
00000100:00000200:0.0:1786176751.096942:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856292992, portal 25
00000400:00000200:2.0:1786176751.096943:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176751.096947:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000400:00000200:2.0:1786176751.096948:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1786176751.096950:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00080000:2.0:1786176751.096951:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176751.096956:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.096959:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176751.096960:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176751.096963:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.096968:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176751.096972:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176751.096974:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000100:00000001:0.0:1786176751.096976:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:2.0:1786176751.096977:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176751.096980:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000040:0.0:1786176751.096984:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000001:2.0:1786176751.096985:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000010:0.0:1786176751.096988:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000020:00000001:2.0:1786176751.096988:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000200:0.0:1786176751.096993:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856292992, offset 0
00000100:00000040:2.0:1786176751.096995:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.096998:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176751.097000:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786176751.097004:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176751.097004:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176751.097007:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176751.097010:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176751.097012:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176751.097014:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786176751.097015:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786176751.097017:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176751.097020:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176751.097021:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176751.097023:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1786176751.097024:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000008:00000001:2.0:1786176751.097027:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176751.097030:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176751.097032:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000010:2.0:1786176751.097034:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000020:00000040:2.0:1786176751.097037:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:0.0:1786176751.097038:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:2.0:1786176751.097042:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176751.097044:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176751.097045:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1786176751.097046:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1786176751.097049:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:2.0:1786176751.097049:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176751.097051:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176751.097055:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254460:254460:256:4294967295] 192.168.202.110@tcp LPNI seq info [254449:254449:8:4294967295]
02000000:00000010:2.0:1786176751.097056:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000400:00000200:0.0:1786176751.097061:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176751.097061:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176751.097068:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176751.097093:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176751.097095:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176751.097100:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1786176751.097102:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1786176751.097112:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176751.097114:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176751.097117:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:0.0:1786176751.097119:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000100:00080000:2.0:1786176751.097120:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176751.097123:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176751.097124:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:0.0:1786176751.097128:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176751.097128:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176751.097132:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6974
00000800:00000200:0.0:1786176751.097134:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000400:00000200:2.0:1786176751.097138:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176751.097144:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:0.1:1786176751.097144:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786176751.097148:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:0.1:1786176751.097150:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:2.0:1786176751.097154:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.097157:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:0.0:1786176751.097160:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786176751.097161:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000010:0.0:1786176751.097163:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000100:00000001:2.0:1786176751.097164:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176751.097168:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176751.097169:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176751.097170:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:0.0:1786176751.097171:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:2.0:1786176751.097173:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176751.097176:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:0.0:1786176751.097177:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:2.0:1786176751.097179:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:0.0:1786176751.097182:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000100:00000040:2.0:1786176751.097184:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:0.0:1786176751.097187:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:2.0:1786176751.097187:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176751.097190:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.097192:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786176751.097193:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:2.0:1786176751.097194:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176751.097200:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:0.0:1786176751.097201:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176751.097202:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176751.097206:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176751.097209:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176751.097213:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176751.097216:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786176751.097219:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000008:00100000:2.0:1786176751.097220:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176751.097224:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176751.097225:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176751.097226:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176751.097231:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000020:00000040:2.0:1786176751.097235:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:0.0:1786176751.097238:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176751.097239:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176751.097241:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.097243:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786176751.097244:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176751.097246:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176751.097247:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.097249:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176751.097251:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000400:00000200:0.0:1786176751.097255:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176751.097256:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.097261:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301650637
00000100:00100000:2.0:1786176751.097264:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:0.0:1786176751.097268:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:2.0:1786176751.097274:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176751.097276:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786176751.097276:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176751.097280:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097283:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301650637
00000100:00000010:0.0:1786176751.097290:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134540.
00000100:00000001:2.0:1786176751.097290:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176751.097294:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786176751.097295:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00080000:2.0:1786176751.097299:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786176751.097301:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786176751.097302:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176751.097305:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176751.097306:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176751.097308:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786176751.097309:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00080000:2.0:1786176751.097312:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6974
00000100:00100000:0.0:1786176751.097316:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176751/real 1786176751]  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786176751.097317:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176751.097323:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176751.097326:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176751.097332:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.097335:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:0.0:1786176751.097336:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:2.0:1786176751.097338:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000001:0.0:1786176751.097339:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176751.097343:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000100:00000001:2.0:1786176751.097343:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176751.097347:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:0.0:1786176751.097348:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000020:00000040:2.0:1786176751.097349:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000400:00000010:0.0:1786176751.097353:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000020:00000001:2.0:1786176751.097353:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176751.097356:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:0.0:1786176751.097358:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:2.0:1786176751.097359:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176751.097363:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000200:0.0:1786176751.097364:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176751.097368:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.097371:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176751.097375:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176751.097379:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176751.097382:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000200:0.0:1786176751.097386:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176751.097386:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176751.097392:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176751.097396:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176751.097399:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176751.097402:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:2.0:1786176751.097402:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.097405:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176751.097406:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176751.097408:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786176751.097410:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000008:00000001:2.0:1786176751.097412:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.097413:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176751.097414:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176751.097419:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786176751.097419:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786176751.097421:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176751.097425:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176751.097427:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176751.097432:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786176751.097434:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786176751.097434:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176751.097437:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176751.097441:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1786176751.097441:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786176751.097446:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.097453:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00100000:2.0:1786176751.097454:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176751.097460:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.097464:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176751.097465:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.097470:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176751.097473:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786176751.097478:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00080000:0.0:1786176751.097480:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:2.0:1786176751.097482:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176751.097486:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000040:2.0:1786176751.097487:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097489:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176751.097498:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176751.097499:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176751.097505:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6984)
00000100:00100000:0.0:1786176751.097530:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856292992:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176751.097536:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.097537:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176751.097541:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856292992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097547:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176751.097549:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176751.097551:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176751.097554:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176751.097555:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176751.097557:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176751.097559:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176751.097561:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0800.
02000000:00000001:0.0:1786176751.097564:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176751.097565:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176751.097567:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:0.0:1786176751.097569:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176751.097570:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.097572:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.097578:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176751.097579:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.097580:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176751.097581:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176751.097582:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176751.097585:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097594:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176751.097595:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176751.097596:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176751.097598:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176751.097599:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176751.097601:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856293120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176751.097603:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176751.097604:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176751.097605:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176751.097606:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176751.097608:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176751.097609:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176751.097612:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176751.097613:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176751.097616:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176751.097618:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856293120, portal 10
00000100:00000001:0.0:1786176751.097619:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176751.097620:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178944 : -114548255372672 : ffff97d1a7edca80)
00000100:00000040:0.0:1786176751.097625:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097629:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176751.097631:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176751.097632:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176751.097634:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856293120, offset 0
00000400:00000200:0.0:1786176751.097637:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176751.097639:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176751.097642:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176751.097644:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176751.097646:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176751.097647:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176751.097649:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176751.097651:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254461:254461:256:4294967295] 192.168.202.110@tcp LPNI seq info [254450:254450:8:4294967295]
00000400:00000200:0.0:1786176751.097654:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176751.097659:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176751.097662:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176751.097664:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176751.097667:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176751.097669:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176751.097672:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176751.097673:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1786176751.097684:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176751.097685:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176751.097687:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176751.097690:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176751.097694:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176751.097696:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176751.097697:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176751.097700:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097705:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176751.097707:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097712:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.097713:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176751.097715:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176751.097718:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176751.097720:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176751.097722:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301646858
00000100:00000001:0.0:1786176751.097724:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301650637
00000100:00000001:0.0:1786176751.097725:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301646858
00000100:00000001:0.0:1786176751.097727:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301650637
00000100:00000001:0.0:1786176751.097728:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176751.097730:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.097731:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176751.097733:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176751/real 1786176751]  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176751.097739:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176751.097740:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176751.097741:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176751.097743:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176751.097744:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176751.097745:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176751.097747:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176751.097752:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097756:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176751.097757:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.097758:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176751.097759:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176751.097761:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097765:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176751.097767:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097772:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176751.097773:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.097774:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176751.097775:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.097776:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176751.097777:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176751.097779:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176751.097781:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176751.097783:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176751.097785:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176751.097790:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856293120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176751.097793:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176751.097795:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097798:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176751.097799:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176751.097800:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176751.097802:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176751.097803:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176751.097804:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856293248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176751.097806:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176751.097807:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176751.097808:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176751.097810:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176751.097812:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176751.097813:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176751.097815:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176751.097817:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176751.097818:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176751.097820:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856293248, portal 10
00000100:00000001:0.0:1786176751.097822:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176751.097822:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000100:00000040:0.0:1786176751.097826:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097831:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176751.097833:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176751.097834:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176751.097835:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856293248, offset 0
00000400:00000200:0.0:1786176751.097838:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176751.097840:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176751.097842:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176751.097844:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176751.097846:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176751.097847:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176751.097849:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176751.097851:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254462:254462:256:4294967295] 192.168.202.110@tcp LPNI seq info [254451:254451:8:4294967295]
00000400:00000200:0.0:1786176751.097854:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176751.097858:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176751.097861:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176751.097863:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176751.097865:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176751.097867:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176751.097870:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176751.097871:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1786176751.097874:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176751.097875:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176751.097877:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176751.097879:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176751.097881:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176751.097882:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176751.097883:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176751.097886:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097891:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176751.097893:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097897:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.097898:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176751.097900:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176751.097902:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176751.097904:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176751.097905:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301646983
00000100:00000001:0.0:1786176751.097907:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301650637
00000100:00000001:0.0:1786176751.097908:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301646983
00000100:00000001:0.0:1786176751.097909:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301650637
00000100:00000001:0.0:1786176751.097911:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176751.097912:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.097913:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176751.097915:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176751/real 1786176751]  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176751.097919:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176751.097920:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176751.097921:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176751.097922:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176751.097923:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176751.097924:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176751.097926:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176751.097932:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097936:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176751.097937:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.097938:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176751.097938:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176751.097940:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097944:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176751.097947:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097952:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176751.097954:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.097955:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176751.097955:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.097956:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176751.097957:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176751.097959:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176751.097961:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176751.097962:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176751.097964:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176751.097969:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856293248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176751.097971:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176751.097973:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.097977:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176751.097978:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176751.097979:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176751.097981:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176751.097982:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176751.097984:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856293376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176751.097985:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176751.097986:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176751.097987:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176751.097988:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176751.097990:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176751.097991:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176751.097993:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176751.097995:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176751.097997:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176751.097999:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856293376, portal 4
00000100:00000001:0.0:1786176751.098000:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176751.098001:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454188800 : -114548255362816 : ffff97d1a7edf100)
00000100:00000040:0.0:1786176751.098005:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098009:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176751.098011:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176751.098012:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176751.098014:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856293376, offset 0
00000400:00000200:0.0:1786176751.098016:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176751.098018:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176751.098020:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176751.098021:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176751.098023:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176751.098024:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176751.098026:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176751.098028:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254463:254463:256:4294967295] 192.168.202.110@tcp LPNI seq info [254452:254452:8:4294967295]
00000400:00000200:0.0:1786176751.098031:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176751.098035:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176751.098038:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176751.098039:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176751.098041:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176751.098043:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176751.098045:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176751.098047:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1786176751.098049:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176751.098051:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176751.098052:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176751.098054:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176751.098056:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176751.098058:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176751.098060:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176751.098062:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098067:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176751.098069:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098085:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.098086:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176751.098088:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176751.098091:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176751.098092:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176751.098094:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301646858
00000100:00000001:0.0:1786176751.098095:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301650637
00000100:00000001:0.0:1786176751.098097:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301646858
00000100:00000001:0.0:1786176751.098098:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301650637
00000100:00000001:0.0:1786176751.098100:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176751.098101:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.098102:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176751.098104:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176751/real 1786176751]  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176751.098109:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176751.098110:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176751.098111:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176751.098113:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176751.098114:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176751.098115:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176751.098117:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176751.098122:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098127:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176751.098128:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.098129:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176751.098130:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176751.098132:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098136:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176751.098139:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098143:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176751.098144:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.098145:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176751.098146:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.098147:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176751.098148:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176751.098149:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176751.098151:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176751.098152:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176751.098154:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176751.098159:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856293376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176751.098161:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176751.098163:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098167:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176751.098168:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176751.098170:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176751.098171:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176751.098173:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176751.098174:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856293504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176751.098177:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176751.098178:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176751.098178:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176751.098179:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176751.098181:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176751.098183:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176751.098185:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176751.098186:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176751.098188:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176751.098190:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856293504, portal 4
00000100:00000001:0.0:1786176751.098191:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176751.098192:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000040:0.0:1786176751.098196:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098200:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176751.098202:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176751.098203:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176751.098204:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856293504, offset 0
00000400:00000200:0.0:1786176751.098207:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176751.098209:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176751.098211:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176751.098212:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176751.098214:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176751.098215:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176751.098217:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176751.098219:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254464:254464:256:4294967295] 192.168.202.110@tcp LPNI seq info [254453:254453:8:4294967295]
00000400:00000200:0.0:1786176751.098221:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176751.098226:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176751.098229:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176751.098231:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176751.098234:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176751.098235:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786176751.098238:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176751.098240:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1786176751.098242:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176751.098244:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176751.098246:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176751.098248:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176751.098249:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176751.098251:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176751.098252:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176751.098254:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098259:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176751.098261:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176776 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098265:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.098266:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176751.098268:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176751.098269:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176751.098271:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176751.098273:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301646866
00000100:00000001:0.0:1786176751.098274:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301650637
00000100:00000001:0.0:1786176751.098276:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301646866
00000100:00000001:0.0:1786176751.098277:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301650637
00000100:00000001:0.0:1786176751.098279:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176751.098280:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.098281:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176751.098283:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176751/real 1786176751]  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176751.098289:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176751.098289:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176751.098291:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176751.098292:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176751.098293:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176751.098294:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176751.098296:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176751.098301:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098305:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176751.098305:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.098307:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176751.098308:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176751.098310:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098314:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176751.098316:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098320:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176751.098321:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.098322:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176751.098323:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.098324:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176751.098325:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176751.098327:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176751.098328:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176751.098330:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176751.098332:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176751.098337:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856293504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176751.098340:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.098341:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176751.098343:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edca80 x1872934856293120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098347:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176751.098348:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176751.098349:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176751.098350:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176751.098351:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176751.098352:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176751.098353:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176751.098354:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3400.
02000000:00000001:0.0:1786176751.098356:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176751.098356:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176751.098357:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edca80.
00000100:00000001:0.0:1786176751.098359:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176751.098360:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.098361:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176751.098363:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856293248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098367:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176751.098368:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176751.098369:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176751.098370:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176751.098371:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176751.098371:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176751.098372:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176751.098373:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:0.0:1786176751.098374:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176751.098375:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176751.098376:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
00000100:00000001:0.0:1786176751.098377:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176751.098378:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.098379:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176751.098381:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf100 x1872934856293376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098385:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176751.098386:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176751.098387:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176751.098388:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176751.098389:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176751.098389:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176751.098390:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176751.098391:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:0.0:1786176751.098392:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176751.098393:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176751.098394:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf100.
00000100:00000001:0.0:1786176751.098395:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176751.098396:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.098397:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176751.098398:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856293504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176776 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176751.098402:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176751.098403:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176751.098404:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176751.098405:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176751.098405:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176751.098406:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176751.098407:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176751.098408:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:0.0:1786176751.098409:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176751.098410:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176751.098411:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:0.0:1786176751.098412:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176751.098412:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.098414:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176751.098416:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176751.098417:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176751.098418:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176751.098420:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176751.672741:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176751.672744:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176751.672747:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176752.696719:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176752.696723:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176752.696727:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176754.744710:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176754.744714:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176754.744718:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176754.936624:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301650637, at 4301650688
00000100:00000010:0.0:1786176754.936638:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000020:00000001:0.0:1786176755.768641:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176755.768646:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176755.768650:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176756.216632:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176756.216641:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176756.216645:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176756.216649:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176756.216655:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176756.216662:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6979
00000400:00000200:2.0:1786176756.216671:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176756.216678:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176756.216681:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176756.216683:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.216686:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176756.216688:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.216691:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176756.216696:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176756.216699:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176756.216702:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176756.216706:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176756.216709:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176756.216713:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176756.216716:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.216719:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176756.216721:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.216724:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176756.216727:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176756.216731:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176756.216735:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176756.216739:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176756.216743:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176756.216746:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176756.216748:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176756.216751:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176756.216762:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176756.216765:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176756.216769:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176756.216770:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176756.216774:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176756.216777:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176756.216782:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176756.216790:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786176756.216797:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176756.216807:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176756.216816:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176756.216820:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176756.216824:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176756.216850:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176756.216853:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176756.216856:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176756.216858:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176756.216859:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176756.216862:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176756.216864:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6979
00000400:00000200:2.0:1786176756.216867:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176756.216869:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176756.216871:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176756.216873:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.216875:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176756.216876:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.216878:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176756.216880:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176756.216881:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176756.216882:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176756.216884:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176756.216886:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176756.216887:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176756.216889:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.216891:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176756.216893:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.216895:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176756.216897:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176756.216898:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176756.216901:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786176756.216901:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176756.216903:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786176756.216908:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176756.216911:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:2.0:1786176756.216911:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176756.216915:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176756.216918:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:0.0:1786176756.216922:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1786176756.216923:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176756.216929:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176756.216934:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176756.216940:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000100:00000001:0.0:1786176756.216941:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176756.216945:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786176756.216945:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176756.216950:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176756.216952:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786176756.216952:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176756.216956:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176756.216957:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176756.216959:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176756.216960:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176756.216964:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176756.216965:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856293632:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176756.216971:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000010:2.0:1786176756.216971:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
02000000:00000001:0.0:1786176756.216975:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176756.216979:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176756.216979:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176756.216985:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:2.0:1786176756.216991:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:0.0:1786176756.216992:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176756.216997:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176756.217007:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176756.217008:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176756.217011:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786176756.217012:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000100:00000040:2.0:1786176756.217017:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1786176756.217035:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000001:2.0:1786176756.217038:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176756.217042:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:0.0:1786176756.217045:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856293632, portal 25
00000100:00080000:2.0:1786176756.217046:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176756.217049:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786176756.217051:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1786176756.217052:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176756.217055:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000001:2.0:1786176756.217056:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176756.217061:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6979
00000100:00000040:0.0:1786176756.217065:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786176756.217068:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176756.217071:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176756.217072:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176756.217078:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217083:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176756.217087:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176756.217089:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786176756.217091:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176756.217095:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000040:0.0:1786176756.217096:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:2.0:1786176756.217099:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000010:0.0:1786176756.217101:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000020:00000001:2.0:1786176756.217102:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176756.217105:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000200:0.0:1786176756.217106:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856293632, offset 0
00000020:00000001:2.0:1786176756.217114:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:0.0:1786176756.217116:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:2.0:1786176756.217117:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176756.217121:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217123:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176756.217124:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217126:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786176756.217130:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786176756.217130:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176756.217135:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217137:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176756.217138:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786176756.217141:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:2.0:1786176756.217143:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176756.217147:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176756.217148:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786176756.217152:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:0.0:1786176756.217155:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00100000:2.0:1786176756.217156:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1786176756.217161:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000008:00000001:2.0:1786176756.217162:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176756.217165:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217167:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176756.217172:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254465:254465:256:4294967295] 192.168.202.110@tcp LPNI seq info [254454:254454:8:4294967295]
00000100:00000010:2.0:1786176756.217176:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000400:00000200:0.0:1786176756.217177:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786176756.217183:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176756.217189:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1786176756.217192:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1786176756.217193:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176756.217198:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176756.217200:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176756.217202:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786176756.217207:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000010:2.0:1786176756.217209:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786176756.217214:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1786176756.217218:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000100:00100000:2.0:1786176756.217219:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176756.217228:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176756.217230:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786176756.217236:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1786176756.217236:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1786176756.217245:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1786176756.217246:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176756.217249:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176756.217253:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176756.217256:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.1:1786176756.217257:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786176756.217259:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:0.1:1786176756.217263:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1786176756.217266:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000800:00000100:0.0:1786176756.217268:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176756.217273:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000100:00080000:2.0:1786176756.217273:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6979
00000400:00000200:0.0:1786176756.217282:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786176756.217282:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176756.217291:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786176756.217291:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176756.217298:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786176756.217301:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176756.217305:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000040:2.0:1786176756.217306:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217309:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786176756.217311:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1786176756.217313:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:0.0:1786176756.217318:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176756.217318:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176756.217325:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176756.217329:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:0.0:1786176756.217333:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000001:2.0:1786176756.217333:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000040:0.0:1786176756.217338:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176756.217338:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176756.217344:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:0.0:1786176756.217352:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176756.217352:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:0.0:1786176756.217358:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:2.0:1786176756.217361:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217363:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786176756.217365:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176756.217366:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217371:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786176756.217374:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786176756.217374:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:0.0:1786176756.217380:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301655757
00000100:00080000:2.0:1786176756.217380:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217386:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176756.217388:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:0.0:1786176756.217389:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
02000000:00000001:2.0:1786176756.217391:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176756.217395:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176756.217396:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786176756.217399:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000008:00100000:2.0:1786176756.217400:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176756.217405:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176756.217406:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.217410:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301655757
00000100:00000010:2.0:1786176756.217411:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786176756.217416:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000010:0.0:1786176756.217417:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134c00.
02000000:00000001:2.0:1786176756.217419:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176756.217421:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.217422:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786176756.217424:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176756.217426:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1786176756.217427:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176756.217428:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.217430:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.217434:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000010:2.0:1786176756.217435:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00100000:0.0:1786176756.217439:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176756/real 1786176756]  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176756.217440:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176756.217448:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176756.217460:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176756.217464:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176756.217470:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176756.217471:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176756.217474:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176756.217478:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000100:00000001:2.0:1786176756.217480:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176756.217482:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000100:00080000:2.0:1786176756.217483:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1786176756.217487:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00080000:2.0:1786176756.217488:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786176756.217491:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1786176756.217491:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176756.217496:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176756.217498:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:0.0:1786176756.217502:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176756.217502:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6979
00000400:00000200:2.0:1786176756.217533:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176756.217536:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176756.217538:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176756.217541:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217542:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176756.217545:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217546:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176756.217548:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176756.217550:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176756.217551:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176756.217553:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176756.217555:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176756.217558:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176756.217561:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217563:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176756.217567:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217572:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176756.217577:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000200:0.0:1786176756.217578:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176756.217582:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176756.217588:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176756.217592:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786176756.217595:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:2.0:1786176756.217596:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176756.217599:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786176756.217599:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176756.217605:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786176756.217606:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000008:00100000:2.0:1786176756.217608:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786176756.217609:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:2.0:1786176756.217612:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176756.217616:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176756.217617:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786176756.217623:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000100:00000001:0.0:1786176756.217627:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786176756.217628:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176756.217632:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:0.0:1786176756.217634:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176756.217635:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176756.217640:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176756.217643:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176756.217646:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.217647:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000010:2.0:1786176756.217650:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:0.0:1786176756.217651:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.217655:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176756.217657:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176756.217657:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.217659:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176756.217662:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00100000:2.0:1786176756.217665:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00080000:0.0:1786176756.217666:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176756.217675:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176756.217680:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1786176756.217680:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176756.217684:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176756.217688:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176756.217692:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176756.217700:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856293632:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176756.217708:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.217714:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786176756.217715:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176756.217718:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856293632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.217725:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000040:2.0:1786176756.217725:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6989)
02000000:00000001:0.0:1786176756.217728:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176756.217732:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176756.217736:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176756.217739:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176756.217742:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176756.217744:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176756.217747:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:0.0:1786176756.217750:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176756.217753:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176756.217755:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:0.0:1786176756.218905:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176756.218908:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.218911:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.218918:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176756.218919:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.218921:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176756.218923:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176756.218925:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176756.218935:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.218943:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176756.218945:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176756.218948:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176756.218950:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176756.218952:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176756.218955:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856293760:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176756.218958:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176756.218962:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176756.218963:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176756.218965:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176756.218974:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176756.218976:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176756.218982:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176756.218985:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176756.218990:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176756.218996:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856293760, portal 10
00000100:00000001:0.0:1786176756.218997:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176756.218999:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:0.0:1786176756.219005:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219011:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176756.219013:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176756.219015:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176756.219018:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856293760, offset 0
00000400:00000200:0.0:1786176756.219037:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176756.219045:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176756.219052:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176756.219056:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176756.219059:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176756.219062:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176756.219065:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176756.219068:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254466:254466:256:4294967295] 192.168.202.110@tcp LPNI seq info [254455:254455:8:4294967295]
00000400:00000200:0.0:1786176756.219070:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176756.219077:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176756.219081:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176756.219085:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176756.219090:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176756.219094:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176756.219098:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176756.219100:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1786176756.219103:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176756.219105:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176756.219109:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176756.219112:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176756.219115:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176756.219117:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176756.219120:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176756.219123:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219130:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176756.219133:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219137:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.219139:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176756.219144:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176756.219148:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176756.219150:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176756.219152:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301651979
00000100:00000001:0.0:1786176756.219155:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301655757
00000100:00000001:0.0:1786176756.219156:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301651979
00000100:00000001:0.0:1786176756.219158:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301655757
00000100:00000001:0.0:1786176756.219160:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176756.219161:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.219163:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176756.219166:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176756/real 1786176756]  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176756.219172:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176756.219173:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176756.219175:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176756.219177:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176756.219178:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176756.219179:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176756.219181:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176756.219187:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219191:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176756.219192:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219193:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176756.219195:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176756.219197:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219202:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176756.219205:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219211:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176756.219213:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219214:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176756.219215:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219217:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176756.219219:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176756.219221:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176756.219224:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176756.219226:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176756.219228:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176756.219234:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856293760:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176756.219237:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176756.219239:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219244:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176756.219245:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176756.219247:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176756.219248:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176756.219249:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176756.219251:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856293888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176756.219253:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176756.219255:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176756.219256:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176756.219257:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176756.219259:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176756.219261:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176756.219263:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176756.219265:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176756.219267:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176756.219269:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856293888, portal 10
00000100:00000001:0.0:1786176756.219270:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176756.219271:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:0.0:1786176756.219275:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219279:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176756.219281:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176756.219282:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176756.219284:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856293888, offset 0
00000400:00000200:0.0:1786176756.219286:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176756.219289:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176756.219291:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176756.219293:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176756.219294:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176756.219296:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176756.219298:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176756.219300:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254467:254467:256:4294967295] 192.168.202.110@tcp LPNI seq info [254456:254456:8:4294967295]
00000400:00000200:0.0:1786176756.219303:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176756.219306:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176756.219309:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176756.219311:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176756.219313:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176756.219315:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176756.219318:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176756.219320:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1786176756.219322:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176756.219323:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176756.219325:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176756.219327:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176756.219330:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176756.219331:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176756.219333:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176756.219335:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219340:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176756.219342:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219346:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.219347:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176756.219349:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176756.219351:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176756.219353:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176756.219354:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301652104
00000100:00000001:0.0:1786176756.219356:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301655757
00000100:00000001:0.0:1786176756.219357:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301652104
00000100:00000001:0.0:1786176756.219358:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301655757
00000100:00000001:0.0:1786176756.219360:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176756.219361:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.219362:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176756.219364:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176756/real 1786176756]  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176756.219369:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176756.219369:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176756.219370:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176756.219372:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176756.219373:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176756.219374:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176756.219376:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176756.219381:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219386:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176756.219387:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219388:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176756.219389:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176756.219391:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219395:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176756.219398:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219402:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176756.219403:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219404:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176756.219405:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219406:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176756.219407:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176756.219409:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176756.219411:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176756.219412:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176756.219414:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176756.219419:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856293888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176756.219421:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176756.219424:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219428:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176756.219429:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176756.219431:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176756.219432:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176756.219433:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176756.219435:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856294016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176756.219437:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176756.219438:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176756.219439:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176756.219440:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176756.219450:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176756.219451:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176756.219454:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176756.219455:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176756.219457:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176756.219459:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856294016, portal 4
00000100:00000001:0.0:1786176756.219460:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176756.219461:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000040:0.0:1786176756.219466:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219470:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176756.219472:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176756.219473:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176756.219474:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856294016, offset 0
00000400:00000200:0.0:1786176756.219477:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176756.219479:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176756.219481:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176756.219483:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176756.219485:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176756.219486:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176756.219488:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176756.219490:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254468:254468:256:4294967295] 192.168.202.110@tcp LPNI seq info [254457:254457:8:4294967295]
00000400:00000200:0.0:1786176756.219493:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176756.219497:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176756.219504:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176756.219506:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176756.219522:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176756.219524:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176756.219527:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176756.219529:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1786176756.219531:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176756.219533:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176756.219537:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176756.219541:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176756.219544:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176756.219547:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176756.219550:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176756.219554:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219561:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176756.219565:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219572:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.219573:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176756.219577:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176756.219580:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176756.219584:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176756.219586:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301651980
00000100:00000001:0.0:1786176756.219589:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301655757
00000100:00000001:0.0:1786176756.219592:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301651980
00000100:00000001:0.0:1786176756.219594:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301655757
00000100:00000001:0.0:1786176756.219596:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176756.219598:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.219601:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176756.219605:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176756/real 1786176756]  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176756.219615:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176756.219616:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176756.219618:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176756.219621:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176756.219623:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176756.219625:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176756.219629:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176756.219639:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219645:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176756.219646:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219648:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176756.219649:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176756.219651:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219656:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176756.219659:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219665:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176756.219668:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219669:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176756.219671:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219673:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176756.219674:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176756.219677:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176756.219681:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176756.219684:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176756.219689:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176756.219698:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856294016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176756.219702:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176756.219706:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219713:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176756.219714:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176756.219716:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176756.219718:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176756.219719:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176756.219721:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856294144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176756.219723:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176756.219725:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176756.219726:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176756.219727:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176756.219730:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176756.219731:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176756.219734:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176756.219735:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176756.219738:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176756.219740:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856294144, portal 4
00000100:00000001:0.0:1786176756.219742:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176756.219743:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:0.0:1786176756.219749:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219756:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176756.219758:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176756.219761:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176756.219764:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856294144, offset 0
00000400:00000200:0.0:1786176756.219768:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176756.219773:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176756.219777:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176756.219779:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176756.219782:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176756.219784:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176756.219787:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176756.219790:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254469:254469:256:4294967295] 192.168.202.110@tcp LPNI seq info [254458:254458:8:4294967295]
00000400:00000200:0.0:1786176756.219794:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176756.219798:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176756.219801:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176756.219804:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176756.219807:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176756.219808:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786176756.219811:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176756.219813:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1786176756.219816:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176756.219817:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176756.219819:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176756.219822:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176756.219824:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176756.219825:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176756.219827:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176756.219829:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219835:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176756.219836:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176781 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219841:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.219842:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176756.219844:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176756.219846:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176756.219848:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176756.219850:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301651988
00000100:00000001:0.0:1786176756.219852:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301655757
00000100:00000001:0.0:1786176756.219853:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301651988
00000100:00000001:0.0:1786176756.219855:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301655757
00000100:00000001:0.0:1786176756.219856:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176756.219857:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.219858:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176756.219860:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176756/real 1786176756]  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176756.219865:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176756.219866:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176756.219867:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176756.219869:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176756.219870:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176756.219871:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176756.219873:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176756.219878:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219882:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176756.219883:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219884:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176756.219885:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176756.219887:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219891:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176756.219893:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219897:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176756.219899:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219900:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176756.219901:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219902:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176756.219904:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176756.219905:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176756.219907:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176756.219909:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176756.219911:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176756.219915:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856294144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176756.219918:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219920:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176756.219922:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856293760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219926:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176756.219928:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176756.219930:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176756.219932:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176756.219934:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176756.219935:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176756.219937:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176756.219939:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:0.0:1786176756.219942:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176756.219943:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176756.219945:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:0.0:1786176756.219946:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176756.219947:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219948:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176756.219950:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856293888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219959:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176756.219960:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176756.219961:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176756.219962:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176756.219963:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176756.219964:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176756.219965:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176756.219966:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:0.0:1786176756.219967:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176756.219968:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176756.219968:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:0.0:1786176756.219970:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176756.219970:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219971:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176756.219973:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856294016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219977:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176756.219977:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176756.219978:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176756.219979:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176756.219980:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176756.219981:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176756.219982:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176756.219983:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:0.0:1786176756.219984:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176756.219984:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176756.219985:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:0.0:1786176756.219987:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176756.219987:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.219988:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176756.219990:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856294144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176756.219994:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176756.219994:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176756.219996:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786176756.219997:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176756.219997:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176756.219998:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176756.220000:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176756.220001:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:0.0:1786176756.220002:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176756.220003:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176756.220003:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:0.0:1786176756.220005:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176756.220005:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.220007:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176756.220009:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176756.220010:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176756.220011:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176756.220012:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176757.816648:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176757.816652:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176757.816655:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176758.840681:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176758.840686:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176758.840689:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176760.056593:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301655757, at 4301655808
00000100:00000010:0.0:1786176760.056607:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000020:00000001:2.0:1786176760.888665:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176760.888668:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176760.888671:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.0:1786176761.208870:0:2384:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00080000:2.0:1786176761.336624:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176761.336634:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176761.336636:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176761.336640:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176761.336644:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176761.336652:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6984
00000400:00000200:2.0:1786176761.336660:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176761.336667:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176761.336670:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176761.336673:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.336676:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176761.336678:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.336681:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176761.336685:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176761.336687:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176761.336690:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176761.336694:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176761.336697:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176761.336700:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176761.336703:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.336704:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176761.336706:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.336707:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176761.336710:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176761.336712:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176761.336713:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176761.336715:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176761.336719:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176761.336721:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176761.336722:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176761.336724:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176761.336732:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786176761.336735:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176761.336738:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176761.336739:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176761.336741:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176761.336743:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176761.336747:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176761.336754:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786176761.336760:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176761.336768:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176761.336776:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176761.336780:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176761.336784:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176761.336802:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176761.336806:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176761.336810:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176761.336811:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176761.336813:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176761.336817:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176761.336820:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6984
00000400:00000200:2.0:1786176761.336824:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176761.336827:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176761.336829:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176761.336832:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.336834:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176761.336837:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.336839:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786176761.336840:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1786176761.336841:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176761.336842:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:0.0:1786176761.336844:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1786176761.336844:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786176761.336846:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1786176761.336846:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176761.336848:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176761.336851:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786176761.336853:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176761.336854:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.336856:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176761.336858:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.336860:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786176761.336862:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176761.336863:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786176761.336863:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176761.336866:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1786176761.336867:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1786176761.336868:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176761.336869:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176761.336870:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176761.336873:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176761.336874:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:0.0:1786176761.336876:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856294272:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1786176761.336876:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176761.336878:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786176761.336881:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:2.0:1786176761.336882:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1786176761.336884:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:2.0:1786176761.336886:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176761.336888:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176761.336891:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000020:00000040:2.0:1786176761.336894:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786176761.336895:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176761.336896:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176761.336898:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1786176761.336898:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176761.336900:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176761.336901:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:0.0:1786176761.336903:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:2.0:1786176761.336903:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176761.336905:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176761.336906:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:2.0:1786176761.336909:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176761.336911:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1786176761.336914:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1786176761.336915:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000100:00000001:2.0:1786176761.336921:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000010:0.0:1786176761.336922:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000001:2.0:1786176761.336923:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176761.336927:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176761.336928:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856294272, portal 25
00000100:00000001:0.0:1786176761.336931:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176761.336932:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187904 : -114548255363712 : ffff97d1a7eded80)
00000100:00000001:2.0:1786176761.336936:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176761.336938:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176761.336939:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176761.336942:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176761.336944:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786176761.336946:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1786176761.336946:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176761.336948:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:0.0:1786176761.336949:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176761.336950:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6984
00000400:00000010:0.0:1786176761.336952:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000400:00000200:2.0:1786176761.336954:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:0.0:1786176761.336955:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856294272, offset 0
00000400:00000200:2.0:1786176761.336957:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176761.336959:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786176761.336962:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786176761.336962:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.336964:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176761.336966:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.336968:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786176761.336970:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1786176761.336970:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176761.336972:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176761.336993:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176761.336995:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176761.336997:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176761.336999:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176761.337003:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.337004:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786176761.337006:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1786176761.337006:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.337008:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176761.337011:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176761.337014:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1786176761.337016:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786176761.337016:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176761.337018:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786176761.337020:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:2.0:1786176761.337021:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176761.337023:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176761.337025:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1786176761.337025:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176761.337027:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1786176761.337028:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000008:00000001:2.0:1786176761.337029:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176761.337031:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176761.337032:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254470:254470:256:4294967295] 192.168.202.110@tcp LPNI seq info [254459:254459:8:4294967295]
00000100:00000010:2.0:1786176761.337035:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000400:00000200:0.0:1786176761.337037:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786176761.337037:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176761.337040:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176761.337041:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176761.337042:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176761.337044:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176761.337046:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176761.337048:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000010:2.0:1786176761.337050:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786176761.337053:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786176761.337055:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00100000:2.0:1786176761.337057:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:0.0:1786176761.337060:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176761.337068:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176761.337071:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000200:0.0:1786176761.337072:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:2.0:1786176761.337074:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176761.337078:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1786176761.337079:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176761.337083:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1786176761.337087:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176761.337087:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176761.337089:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:0.0:1786176761.337091:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000100:00080000:2.0:1786176761.337093:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176761.337094:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176761.337095:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1786176761.337097:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1786176761.337098:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176761.337100:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6984
00000400:00000200:0.0:1786176761.337109:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786176761.337110:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176761.337113:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:0.0:1786176761.337114:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000100:00080000:2.0:1786176761.337116:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:0.0:1786176761.337117:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000040:2.0:1786176761.337119:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176761.337121:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1786176761.337121:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176761.337123:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.337125:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:0.0:1786176761.337126:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786176761.337128:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176761.337129:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176761.337131:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176761.337133:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176761.337135:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:0.0:1786176761.337136:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000001:2.0:1786176761.337137:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786176761.337139:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176761.337140:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.337142:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176761.337144:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.337146:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176761.337149:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176761.337152:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176761.337154:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176761.337155:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337156:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1786176761.337156:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176761.337159:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176761.337160:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337162:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176761.337162:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176761.337165:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:0.0:1786176761.337166:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:2.0:1786176761.337168:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337169:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301660877
00000100:00000001:2.0:1786176761.337169:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176761.337173:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000100:00000010:0.0:1786176761.337174:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000020:00000040:2.0:1786176761.337175:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176761.337177:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176761.337179:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176761.337180:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786176761.337181:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:2.0:1786176761.337181:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176761.337183:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337187:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301660877
02000000:00000010:2.0:1786176761.337187:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786176761.337190:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176761.337191:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134c00.
00000100:00000001:0.0:1786176761.337194:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00100000:2.0:1786176761.337195:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176761.337197:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176761.337200:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337203:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1786176761.337203:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176761.337205:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176761.337207:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176761/real 1786176761]  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176761.337209:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176761.337217:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786176761.337217:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1786176761.337219:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:2.0:1786176761.337220:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1786176761.337222:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000100:00080000:2.0:1786176761.337223:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:0.0:1786176761.337224:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000100:00000001:2.0:1786176761.337225:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:0.0:1786176761.337226:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00080000:2.0:1786176761.337227:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176761.337229:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1786176761.337229:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176761.337232:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6984
00000100:00000200:0.0:1786176761.337233:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786176761.337236:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176761.337239:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176761.337241:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:0.0:1786176761.337244:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176761.337245:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.337247:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176761.337249:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.337251:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786176761.337252:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176761.337254:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1786176761.337254:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176761.337255:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:0.0:1786176761.337256:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000001:2.0:1786176761.337257:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786176761.337259:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786176761.337259:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176761.337261:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176761.337263:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00100000:0.0:1786176761.337264:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176761.337266:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176761.337268:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176761.337271:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176761.337272:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176761.337273:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176761.337275:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000040:0.0:1786176761.337277:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176761.337278:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176761.337280:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176761.337282:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786176761.337285:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:2.0:1786176761.337285:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176761.337287:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337288:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176761.337289:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786176761.337290:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176761.337291:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:2.0:1786176761.337291:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176761.337293:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337294:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786176761.337295:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337296:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000010:2.0:1786176761.337298:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000100:00080000:0.0:1786176761.337299:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000020:00000040:2.0:1786176761.337301:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00080000:0.0:1786176761.337303:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
02000000:00000001:2.0:1786176761.337303:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176761.337305:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337306:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1786176761.337307:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176761.337308:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:0.0:1786176761.337310:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176761.337310:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176761.337313:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786176761.337317:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176761.337321:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856294272:192.168.202.110@tcp:250:kworker.0
00000100:00100000:2.0:1786176761.337321:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176761.337326:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.337328:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786176761.337328:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:0.0:1786176761.337331:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eded80 x1872934856294272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176761.337331:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176761.337334:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176761.337344:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337346:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000040:2.0:1786176761.337348:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (6994)
02000000:00000001:0.0:1786176761.337349:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176761.337351:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176761.337354:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176761.337356:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176761.337358:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176761.337360:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176761.337363:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:0.0:1786176761.337367:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176761.337369:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176761.337371:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eded80.
00000100:00000001:0.0:1786176761.337374:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176761.337376:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.337378:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.337386:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176761.337387:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337389:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176761.337391:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176761.337392:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176761.337395:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337407:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176761.337408:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176761.337409:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176761.337411:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176761.337412:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176761.337414:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856294400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176761.337417:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176761.337418:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176761.337419:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176761.337421:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176761.337423:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176761.337424:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176761.337427:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176761.337429:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176761.337432:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176761.337434:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856294400, portal 10
00000100:00000001:0.0:1786176761.337435:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176761.337436:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454180736 : -114548255370880 : ffff97d1a7edd180)
00000100:00000040:0.0:1786176761.337441:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337446:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176761.337447:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176761.337449:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176761.337450:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856294400, offset 0
00000400:00000200:0.0:1786176761.337453:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176761.337456:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176761.337458:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176761.337461:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176761.337462:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176761.337464:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176761.337466:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176761.337468:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254471:254471:256:4294967295] 192.168.202.110@tcp LPNI seq info [254460:254460:8:4294967295]
00000400:00000200:0.0:1786176761.337470:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176761.337475:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176761.337478:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176761.337481:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176761.337484:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176761.337486:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786176761.337489:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176761.337491:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1786176761.337493:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176761.337494:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176761.337496:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176761.337499:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176761.337501:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176761.337502:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176761.337504:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176761.337507:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337541:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176761.337544:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337548:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337550:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176761.337553:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176761.337556:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176761.337559:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176761.337560:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301657098
00000100:00000001:0.0:1786176761.337579:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301660877
00000100:00000001:0.0:1786176761.337581:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301657098
00000100:00000001:0.0:1786176761.337583:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301660877
00000100:00000001:0.0:1786176761.337584:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176761.337586:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337587:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176761.337589:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176761/real 1786176761]  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176761.337594:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176761.337595:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176761.337597:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176761.337598:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176761.337599:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176761.337601:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176761.337603:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176761.337608:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337617:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176761.337618:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.337619:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176761.337620:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176761.337622:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337626:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176761.337629:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337633:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176761.337634:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.337635:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176761.337636:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.337637:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176761.337639:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176761.337640:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176761.337642:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176761.337644:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176761.337646:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176761.337651:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856294400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176761.337653:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176761.337655:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337659:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176761.337660:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176761.337662:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176761.337663:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176761.337665:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176761.337667:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856294528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176761.337669:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176761.337670:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176761.337671:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176761.337672:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176761.337674:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176761.337675:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176761.337678:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176761.337679:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176761.337681:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176761.337683:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856294528, portal 10
00000100:00000001:0.0:1786176761.337684:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176761.337685:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178048 : -114548255373568 : ffff97d1a7edc700)
00000100:00000040:0.0:1786176761.337689:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337693:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176761.337694:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176761.337696:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176761.337697:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856294528, offset 0
00000400:00000200:0.0:1786176761.337700:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176761.337702:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176761.337705:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176761.337707:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176761.337709:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176761.337710:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176761.337712:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176761.337714:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254472:254472:256:4294967295] 192.168.202.110@tcp LPNI seq info [254461:254461:8:4294967295]
00000400:00000200:0.0:1786176761.337717:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176761.337721:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176761.337724:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176761.337726:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176761.337729:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176761.337731:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786176761.337733:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176761.337735:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1786176761.337738:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176761.337739:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176761.337741:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176761.337743:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176761.337745:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176761.337746:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176761.337748:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176761.337750:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337755:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176761.337757:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337761:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337762:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176761.337763:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176761.337765:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176761.337767:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176761.337769:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301657223
00000100:00000001:0.0:1786176761.337770:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301660877
00000100:00000001:0.0:1786176761.337772:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301657223
00000100:00000001:0.0:1786176761.337773:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301660877
00000100:00000001:0.0:1786176761.337774:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176761.337775:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337776:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176761.337779:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176761/real 1786176761]  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176761.337784:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176761.337785:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176761.337786:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176761.337788:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176761.337788:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176761.337790:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176761.337791:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176761.337797:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337801:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176761.337802:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.337803:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176761.337804:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176761.337806:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337810:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176761.337812:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337816:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176761.337817:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.337818:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176761.337819:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.337820:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176761.337821:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176761.337822:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176761.337824:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176761.337826:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176761.337828:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176761.337834:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856294528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176761.337836:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176761.337838:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337841:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176761.337842:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176761.337844:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176761.337845:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176761.337846:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176761.337847:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856294656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176761.337849:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176761.337850:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176761.337851:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176761.337852:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176761.337854:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176761.337855:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176761.337857:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176761.337858:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176761.337860:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176761.337862:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856294656, portal 4
00000100:00000001:0.0:1786176761.337863:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176761.337864:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454181632 : -114548255369984 : ffff97d1a7edd500)
00000100:00000040:0.0:1786176761.337868:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337872:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176761.337873:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176761.337875:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176761.337876:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856294656, offset 0
00000400:00000200:0.0:1786176761.337879:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176761.337881:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176761.337883:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176761.337885:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176761.337886:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176761.337888:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176761.337889:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176761.337891:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254473:254473:256:4294967295] 192.168.202.110@tcp LPNI seq info [254462:254462:8:4294967295]
00000400:00000200:0.0:1786176761.337894:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176761.337898:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176761.337900:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176761.337902:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176761.337904:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176761.337906:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786176761.337908:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176761.337910:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1786176761.337912:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176761.337914:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176761.337915:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176761.337917:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176761.337919:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176761.337920:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176761.337922:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176761.337924:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337928:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176761.337930:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337934:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337935:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176761.337937:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176761.337938:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176761.337940:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176761.337942:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301657098
00000100:00000001:0.0:1786176761.337944:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301660877
00000100:00000001:0.0:1786176761.337945:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301657098
00000100:00000001:0.0:1786176761.337946:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301660877
00000100:00000001:0.0:1786176761.337948:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176761.337949:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.337950:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176761.337952:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176761/real 1786176761]  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176761.337956:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176761.337957:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176761.337958:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176761.337959:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176761.337960:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176761.337961:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176761.337963:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176761.337969:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.337989:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176761.337990:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.337992:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176761.337994:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176761.337998:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338004:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176761.338008:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338016:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176761.338018:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.338019:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176761.338020:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.338022:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176761.338023:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176761.338024:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176761.338026:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176761.338028:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176761.338030:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176761.338035:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856294656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176761.338037:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176761.338039:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338044:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176761.338045:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176761.338046:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176761.338047:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176761.338048:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176761.338050:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856294784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176761.338052:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176761.338054:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176761.338055:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176761.338056:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176761.338058:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176761.338059:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176761.338062:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176761.338063:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176761.338065:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176761.338067:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856294784, portal 4
00000100:00000001:0.0:1786176761.338069:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176761.338070:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:0.0:1786176761.338074:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338078:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176761.338080:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176761.338082:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176761.338083:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856294784, offset 0
00000400:00000200:0.0:1786176761.338085:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176761.338088:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176761.338090:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176761.338092:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176761.338094:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176761.338095:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176761.338097:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176761.338099:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254474:254474:256:4294967295] 192.168.202.110@tcp LPNI seq info [254463:254463:8:4294967295]
00000400:00000200:0.0:1786176761.338101:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176761.338105:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176761.338108:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176761.338110:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176761.338112:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176761.338114:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786176761.338117:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176761.338118:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1786176761.338121:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176761.338123:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176761.338125:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176761.338127:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176761.338129:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176761.338130:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176761.338132:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176761.338134:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338143:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176761.338145:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338149:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.338150:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176761.338152:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176761.338154:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176761.338155:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176761.338156:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301657106
00000100:00000001:0.0:1786176761.338158:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301660877
00000100:00000001:0.0:1786176761.338159:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301657106
00000100:00000001:0.0:1786176761.338161:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301660877
00000100:00000001:0.0:1786176761.338162:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176761.338164:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.338165:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176761.338167:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176761/real 1786176761]  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176761.338171:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176761.338172:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176761.338173:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176761.338174:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176761.338175:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176761.338177:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176761.338179:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176761.338184:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338187:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176761.338189:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.338190:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176761.338191:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176761.338193:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338197:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176761.338199:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338203:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176761.338205:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.338206:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176761.338206:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.338207:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176761.338208:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176761.338210:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176761.338211:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176761.338213:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176761.338215:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176761.338220:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856294784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176761.338222:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.338223:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176761.338225:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd180 x1872934856294400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338229:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176761.338230:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176761.338231:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176761.338233:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176761.338234:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176761.338235:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176761.338237:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176761.338238:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:0.0:1786176761.338240:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176761.338241:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176761.338242:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd180.
00000100:00000001:0.0:1786176761.338243:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176761.338244:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.338246:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176761.338247:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc700 x1872934856294528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338252:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176761.338252:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176761.338253:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176761.338255:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176761.338255:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176761.338256:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176761.338257:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176761.338258:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:0.0:1786176761.338259:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176761.338260:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176761.338261:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc700.
00000100:00000001:0.0:1786176761.338262:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176761.338263:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.338264:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176761.338265:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd500 x1872934856294656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338269:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176761.338270:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176761.338271:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176761.338272:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176761.338272:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176761.338273:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176761.338274:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176761.338275:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:0.0:1786176761.338276:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176761.338277:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176761.338278:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd500.
00000100:00000001:0.0:1786176761.338279:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176761.338280:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.338281:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176761.338283:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856294784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176761.338287:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176761.338288:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176761.338288:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176761.338289:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176761.338290:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176761.338291:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176761.338292:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176761.338293:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:0.0:1786176761.338294:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176761.338295:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176761.338295:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:0.0:1786176761.338297:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176761.338297:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.338298:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176761.338301:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176761.338301:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176761.338303:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176761.338304:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176761.912692:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176761.912696:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176761.912698:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176763.960681:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176763.960684:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176763.960687:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176764.984684:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176764.984688:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176764.984691:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176765.176612:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301660877, at 4301660928
00000100:00000010:0.0:1786176765.176633:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176766.456838:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176766.456844:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176766.456848:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176766.456859:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d180 x1872934856294912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.456881:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176766.456885:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176766.456893:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176766.456897:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176766.456902:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176766.456910:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856294912:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176766.456921:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176766.456938:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176766.456943:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176766.456948:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176766.456957:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176766.456961:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176766.456974:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176766.456983:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176766.456993:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176766.457004:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856294912, portal 25
00000100:00000001:0.0:1786176766.457011:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176766.457017:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297088 : -114547945254528 : ffff97d1ba69d180)
00000100:00000040:0.0:1786176766.457032:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d180 x1872934856294912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.457060:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176766.457088:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176766.457095:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176766.457100:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856294912, offset 0
00000400:00000200:0.0:1786176766.457120:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176766.457133:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176766.457145:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176766.457154:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176766.457162:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176766.457171:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176766.457180:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176766.457191:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254475:254475:256:4294967295] 192.168.202.110@tcp LPNI seq info [254464:254464:8:4294967295]
00000400:00000200:0.0:1786176766.457207:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176766.457231:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176766.457250:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176766.457259:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176766.457273:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176766.457282:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176766.457294:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176766.457300:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1786176766.457311:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176766.457318:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176766.457328:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176766.457341:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176766.457350:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176766.457355:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176766.457363:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176766.457373:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d180 x1872934856294912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.457394:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176766.457399:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d180 x1872934856294912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.457417:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176766.457422:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176766.457437:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176766.457449:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176766.457456:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301665997
00000100:00000010:0.0:1786176766.457467:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176766.457478:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176766.457490:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301665997
00000100:00000010:0.0:1786176766.457498:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134540.
00000100:00000001:0.0:1786176766.457506:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176766.457557:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176766.457565:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176766.457570:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176766.457578:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176766/real 1786176766]  req@ffff97d1ba69d180 x1872934856294912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176766.457588:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176766.457590:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176766.457593:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176766.457596:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176766.457599:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176766.457602:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176766.457605:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d180 x1872934856294912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176766.457613:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d180 x1872934856294912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.457619:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176766.457621:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.457623:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176766.457626:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176766.457630:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d180 x1872934856294912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.457642:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176766.457647:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d180 x1872934856294912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.457655:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176766.457658:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.457659:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176766.457661:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.457663:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176766.457665:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176766.457667:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176766.457671:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176766.457674:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176766.457678:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d180 x1872934856294912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176766.457687:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856294912:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176766.457690:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.457692:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176766.457696:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d180 x1872934856294912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.457703:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176766.457706:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176766.457709:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176766.457713:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176766.457715:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176766.457716:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176766.457719:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176766.457721:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786176766.457725:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176766.457727:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176766.457729:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d180.
00000100:00000001:0.0:1786176766.457732:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176766.457733:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.457736:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.457742:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176766.457744:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176766.457746:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176766.457748:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176766.457778:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176766.457782:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856295040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.457790:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176766.457792:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176766.457795:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176766.457798:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176766.457801:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176766.457805:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176766.457809:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176766.457812:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176766.457814:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176766.457816:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176766.457821:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176766.457823:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176766.457827:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176766.457830:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176766.457834:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176766.457837:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856295040, portal 10
00000100:00000001:0.0:1786176766.457839:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176766.457841:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:0.0:1786176766.457848:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856295040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.457854:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176766.457872:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176766.457874:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176766.457877:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856295040, offset 0
00000400:00000200:0.0:1786176766.457881:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176766.457886:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176766.457889:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176766.457892:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176766.457895:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176766.457898:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176766.457900:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176766.457903:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254476:254476:256:4294967295] 192.168.202.110@tcp LPNI seq info [254465:254465:8:4294967295]
00000400:00000200:0.0:1786176766.457907:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176766.457913:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176766.457917:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176766.457921:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176766.457940:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176766.457943:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176766.457948:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176766.457950:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1786176766.457954:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176766.457956:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176766.457959:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176766.457963:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176766.457966:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176766.457968:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176766.457970:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176766.457974:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856295040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.457982:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176766.457986:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856295040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.457992:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176766.457993:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176766.458013:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176766.458018:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176766.458021:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176766.458022:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301662218
00000100:00000001:0.0:1786176766.458025:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301665997
00000100:00000001:0.0:1786176766.458027:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301662218
00000100:00000001:0.0:1786176766.458030:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301665997
00000100:00000001:0.0:1786176766.458032:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176766.458034:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176766.458036:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176766.458041:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176766/real 1786176766]  req@ffff97d1ba69e300 x1872934856295040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176766.458055:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176766.458057:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176766.458059:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176766.458061:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176766.458062:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176766.458065:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176766.458068:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856295040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176766.458076:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856295040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458082:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176766.458083:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.458085:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176766.458086:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176766.458089:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856295040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458095:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176766.458099:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856295040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458105:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176766.458108:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.458110:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176766.458111:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.458113:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176766.458115:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176766.458118:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176766.458121:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176766.458124:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176766.458128:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856295040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176766.458136:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176766.458140:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176766.458144:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856295168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458149:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176766.458151:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176766.458152:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176766.458154:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176766.458156:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176766.458159:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176766.458162:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176766.458164:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176766.458165:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176766.458167:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176766.458170:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176766.458172:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176766.458175:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176766.458177:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176766.458180:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176766.458183:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856295168, portal 10
00000100:00000001:0.0:1786176766.458185:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176766.458186:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:0.0:1786176766.458192:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856295168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458198:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176766.458201:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176766.458203:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176766.458206:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856295168, offset 0
00000400:00000200:0.0:1786176766.458210:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176766.458214:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176766.458217:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176766.458220:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176766.458222:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176766.458225:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176766.458227:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176766.458230:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254477:254477:256:4294967295] 192.168.202.110@tcp LPNI seq info [254466:254466:8:4294967295]
00000400:00000200:0.0:1786176766.458234:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176766.458240:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176766.458244:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176766.458247:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176766.458250:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176766.458253:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176766.458257:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176766.458259:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1786176766.458263:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176766.458265:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176766.458268:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176766.458271:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176766.458274:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176766.458276:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176766.458279:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176766.458282:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856295168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458290:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176766.458293:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856295168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458299:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176766.458301:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176766.458321:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176766.458324:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176766.458327:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176766.458329:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301662343
00000100:00000001:0.0:1786176766.458332:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301665997
00000100:00000001:0.0:1786176766.458334:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301662343
00000100:00000001:0.0:1786176766.458336:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301665997
00000100:00000001:0.0:1786176766.458338:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176766.458340:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176766.458342:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176766.458345:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176766/real 1786176766]  req@ffff97d1ba69c700 x1872934856295168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176766.458353:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176766.458354:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176766.458356:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176766.458358:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176766.458360:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176766.458362:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176766.458365:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856295168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176766.458374:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856295168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458380:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176766.458381:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.458383:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176766.458384:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176766.458388:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856295168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458394:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176766.458397:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856295168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458404:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176766.458406:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.458408:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176766.458409:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.458411:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176766.458413:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176766.458416:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176766.458418:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176766.458420:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176766.458423:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856295168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176766.458431:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176766.458435:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176766.458438:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856295296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458443:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176766.458445:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176766.458446:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176766.458448:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176766.458450:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176766.458453:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176766.458455:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176766.458458:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176766.458459:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176766.458461:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176766.458464:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176766.458466:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176766.458468:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176766.458471:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176766.458474:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176766.458477:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856295296, portal 4
00000100:00000001:0.0:1786176766.458479:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176766.458480:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:0.0:1786176766.458486:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856295296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458493:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176766.458495:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176766.458498:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176766.458500:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856295296, offset 0
00000400:00000200:0.0:1786176766.458504:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176766.458508:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176766.458529:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176766.458531:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176766.458534:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176766.458536:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176766.458538:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176766.458541:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254478:254478:256:4294967295] 192.168.202.110@tcp LPNI seq info [254467:254467:8:4294967295]
00000400:00000200:0.0:1786176766.458545:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176766.458551:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176766.458555:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176766.458559:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176766.458562:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176766.458565:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176766.458569:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176766.458572:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1786176766.458575:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176766.458577:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176766.458581:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176766.458584:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176766.458587:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176766.458589:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176766.458592:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176766.458596:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856295296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458609:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176766.458612:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856295296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458618:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176766.458619:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176766.458622:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176766.458625:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176766.458628:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176766.458631:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301662219
00000100:00000001:0.0:1786176766.458633:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301665997
00000100:00000001:0.0:1786176766.458636:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301662219
00000100:00000001:0.0:1786176766.458638:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301665997
00000100:00000001:0.0:1786176766.458640:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176766.458642:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176766.458644:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176766.458648:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176766/real 1786176766]  req@ffff97d1ba69c000 x1872934856295296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176766.458656:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176766.458657:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176766.458659:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176766.458661:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176766.458663:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176766.458665:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176766.458668:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856295296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176766.458676:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856295296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458682:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176766.458684:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.458685:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176766.458687:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176766.458690:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856295296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458696:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176766.458700:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856295296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458705:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176766.458707:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.458709:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176766.458710:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.458712:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176766.458714:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176766.458717:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176766.458720:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176766.458722:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176766.458726:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856295296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176766.458735:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176766.458738:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176766.458742:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856295424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458749:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176766.458752:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176766.458754:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176766.458756:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176766.458759:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176766.458762:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176766.458765:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176766.458767:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176766.458769:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176766.458771:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176766.458774:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176766.458776:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176766.458778:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176766.458781:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176766.458783:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176766.458786:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856295424, portal 4
00000100:00000001:0.0:1786176766.458788:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176766.458789:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:0.0:1786176766.458795:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856295424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458801:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176766.458820:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176766.458823:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176766.458825:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856295424, offset 0
00000400:00000200:0.0:1786176766.458833:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176766.458837:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176766.458840:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176766.458843:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176766.458845:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176766.458848:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176766.458850:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176766.458853:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254479:254479:256:4294967295] 192.168.202.110@tcp LPNI seq info [254468:254468:8:4294967295]
00000400:00000200:0.0:1786176766.458858:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176766.458864:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176766.458868:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176766.458871:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176766.458875:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176766.458878:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1786176766.458882:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176766.458884:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:0.0:1786176766.458888:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176766.458890:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176766.458893:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176766.458896:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176766.458899:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176766.458901:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176766.458903:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176766.458907:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856295424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458914:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176766.458917:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856295424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458939:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176766.458942:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176766.458945:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176766.458948:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176766.458951:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176766.458953:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301662227
00000100:00000001:0.0:1786176766.458956:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301665997
00000100:00000001:0.0:1786176766.458958:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301662227
00000100:00000001:0.0:1786176766.458960:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301665997
00000100:00000001:0.0:1786176766.458962:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176766.458964:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176766.458966:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176766.458970:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176766/real 1786176766]  req@ffff97d1ba69ea00 x1872934856295424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176766.458978:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176766.458980:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176766.458982:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176766.458984:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176766.458985:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176766.458987:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176766.458989:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856295424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176766.458994:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856295424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.458999:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176766.459000:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.459001:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176766.459002:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176766.459005:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856295424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.459009:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176766.459012:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856295424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.459016:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176766.459017:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.459018:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176766.459019:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.459020:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176766.459021:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176766.459023:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176766.459025:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176766.459026:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176766.459028:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856295424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176766.459037:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176766.459039:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.459041:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176766.459043:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856295040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.459047:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176766.459048:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176766.459049:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176766.459050:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176766.459052:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176766.459053:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176766.459054:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176766.459055:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:0.0:1786176766.459057:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176766.459058:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176766.459059:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:0.0:1786176766.459061:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176766.459062:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.459063:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176766.459065:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856295168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.459069:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176766.459069:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176766.459070:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176766.459072:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176766.459072:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176766.459073:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176766.459074:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176766.459075:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786176766.459076:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176766.459077:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176766.459078:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:0.0:1786176766.459080:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176766.459080:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.459082:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176766.459084:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856295296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.459088:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176766.459088:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176766.459089:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176766.459090:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176766.459091:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176766.459092:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176766.459093:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176766.459094:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786176766.459095:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176766.459096:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176766.459097:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:0.0:1786176766.459098:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176766.459098:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.459099:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176766.459101:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856295424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176766.459105:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176766.459105:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176766.459106:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176766.459107:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176766.459108:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176766.459109:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176766.459110:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176766.459110:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786176766.459112:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176766.459112:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176766.459113:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:0.0:1786176766.459114:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176766.459115:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.459116:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176766.459119:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176766.459120:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176766.459121:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176766.459123:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176767.032656:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176767.032660:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176767.032662:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176768.056677:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176768.056681:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176768.056683:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176770.104739:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176770.104744:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176770.104748:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176770.296630:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301665997, at 4301666048
00000100:00000010:0.0:1786176770.296645:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000020:00000001:0.0:1786176771.128689:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176771.128693:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176771.128695:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176771.576629:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176771.576637:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176771.576640:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176771.576644:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176771.576650:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176771.576658:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 6994
00000400:00000200:2.0:1786176771.576667:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176771.576674:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176771.576676:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176771.576679:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.576682:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176771.576684:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.576687:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176771.576690:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176771.576691:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176771.576694:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176771.576696:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176771.576698:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176771.576699:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176771.576702:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.576703:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176771.576705:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.576706:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176771.576708:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176771.576710:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176771.576711:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176771.576714:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176771.576716:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176771.576718:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.576720:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176771.576721:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176771.576729:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786176771.576732:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176771.576735:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176771.576736:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.576738:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176771.576741:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176771.576744:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176771.576750:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786176771.576755:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176771.576762:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176771.576775:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176771.576779:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176771.576782:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176771.576801:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176771.576804:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176771.576806:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176771.576807:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176771.576808:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176771.576811:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176771.576812:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6994
00000400:00000200:2.0:1786176771.576815:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176771.576817:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176771.576818:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176771.576820:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.576822:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176771.576829:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.576830:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176771.576832:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176771.576833:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176771.576837:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176771.576838:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176771.576839:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176771.576841:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176771.576843:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.576844:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176771.576845:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.576846:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176771.576849:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176771.576853:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176771.576855:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176771.576856:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176771.576861:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176771.576862:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.576864:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176771.576867:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176771.576870:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.576871:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176771.576883:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000100:00000001:0.0:1786176771.576885:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1786176771.576885:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176771.576890:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1786176771.576891:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1786176771.576892:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.576894:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786176771.576895:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176771.576898:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176771.576899:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176771.576904:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1786176771.576904:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786176771.576909:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176771.576918:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176771.576919:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176771.576923:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176771.576929:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1786176771.576930:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176771.576933:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176771.576935:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1786176771.576936:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786176771.576942:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.576947:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176771.576952:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856295552:192.168.202.110@tcp:250:kworker.0
00000100:00080000:2.0:1786176771.576952:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786176771.576962:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:2.0:1786176771.576965:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
02000000:00000001:0.0:1786176771.576967:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1786176771.576967:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176771.576971:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:0.0:1786176771.576972:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.576973:0:62275:0:(import.c:485:import_select_connection()) Process entered
02000000:00000001:0.0:1786176771.576978:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:2.0:1786176771.576979:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6994
00000400:00000200:2.0:1786176771.576983:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176771.576985:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000010:0.0:1786176771.576987:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
00000100:00080000:2.0:1786176771.576987:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
02000000:00000001:0.0:1786176771.576992:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176771.576994:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.576997:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176771.576998:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786176771.577001:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176771.577001:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176771.577005:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176771.577007:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000010:0.0:1786176771.577008:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000020:00000001:2.0:1786176771.577009:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176771.577013:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176771.577015:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000010:0.0:1786176771.577016:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000020:00000001:2.0:1786176771.577017:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176771.577022:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.577024:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176771.577025:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:0.0:1786176771.577026:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856295552, portal 25
00000100:00000001:2.0:1786176771.577028:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786176771.577030:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000020:00000040:2.0:1786176771.577032:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000001:0.0:1786176771.577033:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00080000:2.0:1786176771.577036:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176771.577039:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176771.577041:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1786176771.577044:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176771.577045:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176771.577049:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.577053:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176771.577055:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176771.577059:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.577063:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.577068:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000010:2.0:1786176771.577069:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786176771.577073:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000040:0.0:1786176771.577077:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:2.0:1786176771.577077:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176771.577080:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.577083:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176771.577084:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:0.0:1786176771.577085:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
02000000:00000001:2.0:1786176771.577085:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786176771.577092:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856295552, offset 0
02000000:00000010:2.0:1786176771.577093:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786176771.577098:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176771.577104:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1786176771.577105:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176771.577113:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176771.577115:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176771.577120:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786176771.577120:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176771.577128:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176771.577130:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1786176771.577133:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:2.0:1786176771.577133:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176771.577137:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176771.577140:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1786176771.577144:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786176771.577149:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1786176771.577151:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:2.0:1786176771.577154:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6994
00000400:00000200:2.0:1786176771.577160:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176771.577161:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786176771.577165:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:0.0:1786176771.577168:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:2.0:1786176771.577169:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176771.577174:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786176771.577177:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254480:254480:256:4294967295] 192.168.202.110@tcp LPNI seq info [254469:254469:8:4294967295]
00000100:00000001:2.0:1786176771.577178:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176771.577183:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.577186:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786176771.577187:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1786176771.577189:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176771.577191:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176771.577192:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176771.577194:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176771.577198:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:0.0:1786176771.577202:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000001:2.0:1786176771.577203:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176771.577210:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.577212:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.0:1786176771.577215:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786176771.577217:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.577220:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000010:0.0:1786176771.577222:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000020:00000040:2.0:1786176771.577223:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176771.577228:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176771.577230:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176771.577232:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.0:1786176771.577235:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1786176771.577236:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176771.577241:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786176771.577242:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:2.0:1786176771.577243:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176771.577246:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176771.577249:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176771.577251:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176771.577252:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786176771.577257:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000010:2.0:1786176771.577258:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176771.577262:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000800:00000100:0.0:1786176771.577263:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1786176771.577264:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000010:0.0:1786176771.577267:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
02000000:00000001:2.0:1786176771.577267:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.577271:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176771.577272:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1786176771.577274:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1786176771.577274:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176771.577278:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000400:00000200:0.0:1786176771.577280:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1786176771.577282:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176771.577285:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176771.577289:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00100000:2.0:1786176771.577290:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176771.577295:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176771.577301:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176771.577303:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176771.577306:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.577314:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176771.577314:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176771.577322:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176771.577325:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176771.577330:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786176771.577337:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176771.577337:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176771.577338:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786176771.577342:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:2.0:1786176771.577342:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176771.577346:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176771.577349:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 6994
00000100:00000001:0.0:1786176771.577351:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786176771.577352:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176771.577358:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176771.577361:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786176771.577362:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786176771.577365:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.577368:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786176771.577369:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301671117
00000100:00000040:2.0:1786176771.577370:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.577375:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176771.577377:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176771.577378:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000010:0.0:1786176771.577379:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000020:00000001:2.0:1786176771.577379:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176771.577381:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176771.577385:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176771.577386:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176771.577388:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176771.577389:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786176771.577390:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176771.577393:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176771.577396:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176771.577398:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:0.0:1786176771.577400:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301671117
00000100:00080000:2.0:1786176771.577401:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176771.577405:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176771.577406:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134c60.
02000000:00000001:2.0:1786176771.577407:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176771.577409:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176771.577411:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.577412:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176771.577413:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176771.577415:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.577416:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176771.577419:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786176771.577422:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:0.0:1786176771.577424:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:2.0:1786176771.577424:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176771.577426:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176771.577429:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786176771.577430:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176771.577431:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176771.577433:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.577435:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.577438:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000010:2.0:1786176771.577438:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786176771.577443:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176771.577445:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176771/real 1786176771]  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786176771.577450:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:0.0:1786176771.577461:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786176771.577463:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000001:0.0:1786176771.577464:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:2.0:1786176771.577467:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786176771.577468:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000100:00000040:2.0:1786176771.577472:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786176771.577473:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176771.577477:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176771.577481:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176771.577486:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176771.577492:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786176771.577499:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176771.577500:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7005)
00000100:00000001:0.0:1786176771.577508:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176771.577533:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.577535:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176771.577537:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176771.577540:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.577546:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176771.577549:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.577554:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176771.577555:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.577556:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176771.577557:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.577559:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176771.577560:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176771.577562:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176771.577564:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176771.577566:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176771.577569:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176771.577574:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856295552:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176771.577577:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.577579:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176771.577581:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856295552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.577585:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176771.577587:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176771.577589:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176771.577591:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176771.577592:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176771.577593:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176771.577595:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176771.577597:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:0.0:1786176771.577600:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176771.577601:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176771.577603:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:0.0:1786176771.577605:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176771.577606:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.577607:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.577613:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176771.577614:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.577615:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176771.577616:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176771.577617:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176771.577620:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.577625:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176771.577626:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176771.577627:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176771.577628:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176771.577630:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176771.577631:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295680:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176771.577633:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176771.577635:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176771.577636:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176771.577637:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176771.577639:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176771.577641:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176771.577643:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176771.577645:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176771.577648:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176771.577650:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856295680, portal 10
00000100:00000001:0.0:1786176771.577651:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176771.577652:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:0.0:1786176771.577657:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.577662:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176771.577663:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176771.577665:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176771.577667:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856295680, offset 0
00000400:00000200:0.0:1786176771.577670:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176771.577672:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176771.577675:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176771.577677:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176771.577679:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176771.577681:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176771.577683:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176771.577685:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254481:254481:256:4294967295] 192.168.202.110@tcp LPNI seq info [254470:254470:8:4294967295]
00000400:00000200:0.0:1786176771.577687:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176771.577692:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176771.577696:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176771.577698:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176771.577700:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176771.577702:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176771.577705:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176771.577707:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1786176771.577709:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176771.577711:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176771.577713:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176771.577715:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176771.577717:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176771.577718:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176771.577720:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176771.577722:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.577728:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176771.577730:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.577734:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.577735:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176771.577738:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176771.577741:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176771.577744:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176771.577746:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301667338
00000100:00000001:0.0:1786176771.577749:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301671117
00000100:00000001:0.0:1786176771.577751:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301667338
00000100:00000001:0.0:1786176771.577754:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301671117
00000100:00000001:0.0:1786176771.577756:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176771.577758:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.577760:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176771.577764:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176771/real 1786176771]  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176771.577771:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176771.577772:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176771.577774:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176771.577776:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176771.577778:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176771.577780:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176771.577783:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176771.577790:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.577796:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176771.577798:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.577800:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176771.577801:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176771.577805:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.577811:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176771.577814:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.577821:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176771.577823:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.577824:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176771.577826:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.577827:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176771.577829:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176771.577831:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176771.577834:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176771.577836:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176771.577840:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176771.577849:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295680:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176771.577853:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176771.577857:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.577863:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176771.577864:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176771.577866:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176771.577867:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176771.577869:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176771.577871:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176771.577888:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176771.577890:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176771.577891:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176771.577893:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176771.577896:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176771.577897:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176771.577900:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176771.577902:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176771.577904:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176771.577907:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856295808, portal 10
00000100:00000001:0.0:1786176771.577908:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176771.577909:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:0.0:1786176771.577914:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.577918:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176771.577920:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176771.577922:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176771.577924:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856295808, offset 0
00000400:00000200:0.0:1786176771.577929:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176771.577933:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176771.577936:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176771.577939:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176771.577942:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176771.577945:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176771.577947:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176771.577950:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254482:254482:256:4294967295] 192.168.202.110@tcp LPNI seq info [254471:254471:8:4294967295]
00000400:00000200:0.0:1786176771.577955:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176771.577962:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176771.577967:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176771.577970:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176771.577974:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176771.577977:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176771.577980:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176771.577982:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1786176771.577985:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176771.577986:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176771.577989:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176771.577991:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176771.577993:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176771.577995:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176771.577997:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176771.578000:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578006:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176771.578008:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578012:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.578013:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176771.578015:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176771.578017:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176771.578020:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176771.578021:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301667463
00000100:00000001:0.0:1786176771.578023:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301671117
00000100:00000001:0.0:1786176771.578024:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301667463
00000100:00000001:0.0:1786176771.578026:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301671117
00000100:00000001:0.0:1786176771.578027:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176771.578028:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.578030:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176771.578032:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176771/real 1786176771]  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176771.578037:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176771.578038:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176771.578039:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176771.578041:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176771.578042:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176771.578043:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176771.578046:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176771.578055:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578061:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176771.578063:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578071:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176771.578073:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176771.578077:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578085:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176771.578089:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578097:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176771.578099:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578101:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176771.578102:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578103:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176771.578104:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176771.578106:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176771.578109:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176771.578110:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176771.578115:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176771.578123:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176771.578127:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176771.578130:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578138:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176771.578139:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176771.578142:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176771.578144:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176771.578146:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176771.578150:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176771.578153:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176771.578155:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176771.578157:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176771.578158:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176771.578161:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176771.578163:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176771.578166:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176771.578168:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176771.578171:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176771.578173:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856295936, portal 4
00000100:00000001:0.0:1786176771.578174:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176771.578175:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:0.0:1786176771.578181:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578185:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176771.578187:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176771.578188:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176771.578190:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856295936, offset 0
00000400:00000200:0.0:1786176771.578192:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176771.578195:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176771.578197:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176771.578199:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176771.578201:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176771.578203:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176771.578204:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176771.578206:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254483:254483:256:4294967295] 192.168.202.110@tcp LPNI seq info [254472:254472:8:4294967295]
00000400:00000200:0.0:1786176771.578209:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176771.578213:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176771.578217:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176771.578219:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176771.578221:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176771.578223:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176771.578226:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176771.578228:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1786176771.578230:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176771.578231:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176771.578234:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176771.578236:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176771.578238:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176771.578239:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176771.578241:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176771.578243:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578248:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176771.578250:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578254:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.578256:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176771.578258:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176771.578260:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176771.578262:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176771.578263:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301667338
00000100:00000001:0.0:1786176771.578265:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301671117
00000100:00000001:0.0:1786176771.578266:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301667338
00000100:00000001:0.0:1786176771.578268:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301671117
00000100:00000001:0.0:1786176771.578269:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176771.578270:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.578271:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176771.578274:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176771/real 1786176771]  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176771.578279:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176771.578280:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176771.578281:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176771.578282:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176771.578283:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176771.578285:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176771.578286:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176771.578291:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578295:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176771.578296:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578298:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176771.578299:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176771.578301:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578305:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176771.578307:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578311:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176771.578313:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578313:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176771.578314:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578315:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176771.578316:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176771.578318:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176771.578320:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176771.578321:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176771.578323:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176771.578329:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856295936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176771.578331:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176771.578333:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578337:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176771.578338:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176771.578339:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176771.578340:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176771.578341:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176771.578344:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856296064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176771.578347:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176771.578348:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176771.578350:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176771.578351:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176771.578355:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176771.578356:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176771.578359:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176771.578362:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176771.578365:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176771.578367:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856296064, portal 4
00000100:00000001:0.0:1786176771.578370:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176771.578371:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000040:0.0:1786176771.578377:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578385:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176771.578387:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176771.578390:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176771.578392:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856296064, offset 0
00000400:00000200:0.0:1786176771.578397:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176771.578401:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176771.578404:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176771.578406:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176771.578407:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176771.578409:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176771.578411:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176771.578413:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254484:254484:256:4294967295] 192.168.202.110@tcp LPNI seq info [254473:254473:8:4294967295]
00000400:00000200:0.0:1786176771.578416:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176771.578420:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176771.578423:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176771.578425:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176771.578428:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176771.578429:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1786176771.578432:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176771.578434:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1786176771.578437:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176771.578438:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176771.578440:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176771.578442:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176771.578445:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176771.578446:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176771.578447:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176771.578450:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578455:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176771.578457:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578461:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.578462:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176771.578464:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176771.578467:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176771.578469:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176771.578471:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301667346
00000100:00000001:0.0:1786176771.578472:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301671117
00000100:00000001:0.0:1786176771.578474:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301667346
00000100:00000001:0.0:1786176771.578475:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301671117
00000100:00000001:0.0:1786176771.578476:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176771.578478:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.578479:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176771.578481:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176771/real 1786176771]  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176771.578486:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176771.578487:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176771.578488:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176771.578489:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176771.578490:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176771.578492:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176771.578494:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176771.578499:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578503:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176771.578503:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578505:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176771.578505:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176771.578508:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578528:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176771.578531:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578535:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176771.578537:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578538:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176771.578539:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578540:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176771.578541:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176771.578543:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176771.578545:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176771.578546:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176771.578549:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176771.578553:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856296064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176771.578556:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578557:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176771.578559:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856295680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578563:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176771.578564:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176771.578566:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176771.578567:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176771.578568:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176771.578569:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176771.578570:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176771.578572:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:0.0:1786176771.578574:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176771.578575:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176771.578576:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:0.0:1786176771.578578:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176771.578578:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578579:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176771.578581:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856295808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578590:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176771.578591:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176771.578592:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176771.578594:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176771.578594:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176771.578595:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176771.578596:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176771.578597:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:0.0:1786176771.578598:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176771.578599:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176771.578600:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:0.0:1786176771.578601:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176771.578602:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578603:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176771.578604:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856295936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578608:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176771.578609:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176771.578610:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176771.578611:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176771.578611:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176771.578612:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176771.578613:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176771.578614:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:0.0:1786176771.578615:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176771.578616:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176771.578616:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:0.0:1786176771.578618:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176771.578618:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578619:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176771.578621:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856296064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176771.578625:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176771.578625:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176771.578627:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176771.578628:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176771.578629:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176771.578629:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176771.578631:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176771.578632:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:0.0:1786176771.578633:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176771.578633:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176771.578634:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:0.0:1786176771.578635:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176771.578636:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578637:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176771.578640:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176771.578641:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176771.578642:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176771.578643:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176773.176764:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176773.176769:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176773.176772:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176774.200663:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176774.200675:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176774.200679:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176775.416615:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301671117, at 4301671168
00000100:00000010:0.0:1786176775.416630:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000020:00000001:2.0:1786176776.248699:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176776.248703:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176776.248705:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176776.696628:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176776.696637:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176776.696639:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176776.696642:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176776.696646:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176776.696652:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7000
00000400:00000200:2.0:1786176776.696660:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176776.696666:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176776.696669:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176776.696672:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.696674:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176776.696676:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.696679:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176776.696685:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176776.696688:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176776.696693:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176776.696697:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176776.696700:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176776.696704:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176776.696710:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.696715:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176776.696721:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.696724:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176776.696726:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176776.696729:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176776.696731:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176776.696733:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176776.696735:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176776.696738:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176776.696739:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176776.696749:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176776.696757:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000020:00000040:2.0:1786176776.696760:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176776.696763:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176776.696764:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176776.696767:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176776.696769:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176776.696773:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176776.696779:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786176776.696785:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176776.696792:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176776.696803:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176776.696807:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176776.696810:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176776.696841:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176776.696844:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176776.696846:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176776.696847:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176776.696848:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176776.696850:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176776.696852:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7000
00000400:00000200:2.0:1786176776.696856:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176776.696858:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176776.696860:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176776.696862:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.696864:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176776.696866:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.696869:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176776.696870:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176776.696871:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176776.696873:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786176776.696874:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1786176776.696874:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176776.696878:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786176776.696879:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1786176776.696881:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786176776.696883:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786176776.696885:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.696888:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176776.696893:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176776.696893:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.696897:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176776.696901:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176776.696906:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176776.696911:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.696914:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1786176776.696914:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786176776.696918:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176776.696922:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786176776.696922:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176776.696926:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176776.696927:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176776.696933:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176776.696933:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176776.696938:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00100000:0.0:1786176776.696939:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856296192:192.168.202.110@tcp:250:kworker.0
00000008:00000001:2.0:1786176776.696942:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.696945:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1786176776.696946:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176776.696950:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:2.0:1786176776.696950:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
02000000:00000001:0.0:1786176776.696953:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786176776.696954:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786176776.696959:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1786176776.696960:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176776.696963:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176776.696964:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176776.696965:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:0.0:1786176776.696966:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:2.0:1786176776.696967:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176776.696969:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176776.696974:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000400:00000200:0.0:1786176776.696978:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176776.696979:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786176776.696983:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000100:00100000:2.0:1786176776.696986:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1786176776.696990:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000001:2.0:1786176776.696996:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176776.696999:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786176776.697000:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856296192, portal 25
00000100:00000040:2.0:1786176776.697003:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.697004:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176776.697007:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454188800 : -114548255362816 : ffff97d1a7edf100)
00000100:00000001:2.0:1786176776.697016:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176776.697017:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176776.697018:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176776.697025:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176776.697028:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176776.697031:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176776.697035:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786176776.697036:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176776.697042:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7000
00000100:00000040:0.0:1786176776.697044:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176776.697048:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000400:00000200:2.0:1786176776.697048:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:0.0:1786176776.697053:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856296192, offset 0
00000400:00000200:2.0:1786176776.697054:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176776.697058:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176776.697061:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786176776.697065:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176776.697066:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176776.697070:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.697072:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176776.697075:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:0.0:1786176776.697076:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786176776.697077:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176776.697079:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176776.697081:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:0.0:1786176776.697084:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:2.0:1786176776.697084:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176776.697088:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:0.0:1786176776.697093:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1786176776.697093:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.697098:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176776.697099:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.697102:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786176776.697103:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:2.0:1786176776.697105:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176776.697109:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1786176776.697112:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1786176776.697113:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176776.697115:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176776.697119:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1786176776.697121:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:2.0:1786176776.697122:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176776.697127:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176776.697128:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176776.697132:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176776.697133:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254485:254485:256:4294967295] 192.168.202.110@tcp LPNI seq info [254474:254474:8:4294967295]
00000100:00000001:2.0:1786176776.697134:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176776.697139:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000400:00000200:0.0:1786176776.697142:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786176776.697142:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176776.697145:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176776.697147:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176776.697149:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176776.697150:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176776.697151:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176776.697156:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000400:00000200:0.0:1786176776.697157:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786176776.697159:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786176776.697167:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00100000:2.0:1786176776.697167:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:0.0:1786176776.697174:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000100:00000001:2.0:1786176776.697175:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176776.697180:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176776.697182:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1786176776.697187:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176776.697194:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1786176776.697195:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176776.697199:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1786176776.697203:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786176776.697204:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176776.697207:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176776.697210:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:0.1:1786176776.697210:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:2.0:1786176776.697216:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000100:0.0:1786176776.697219:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176776.697220:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7000
00000800:00000010:0.0:1786176776.697223:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:2.0:1786176776.697228:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176776.697230:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176776.697231:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176776.697233:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.697234:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176776.697235:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.697238:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176776.697240:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176776.697241:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000400:00000200:0.0:1786176776.697242:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:2.0:1786176776.697243:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176776.697247:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000400:00000200:0.0:1786176776.697252:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:2.0:1786176776.697252:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176776.697256:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:0.0:1786176776.697259:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000100:00000040:2.0:1786176776.697261:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:0.0:1786176776.697264:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:2.0:1786176776.697264:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176776.697268:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176776.697271:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176776.697276:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176776.697279:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176776.697284:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176776.697289:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176776.697292:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786176776.697292:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176776.697295:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1786176776.697298:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176776.697298:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176776.697301:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176776.697304:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176776.697307:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786176776.697312:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1786176776.697312:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176776.697314:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.697316:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000010:2.0:1786176776.697318:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000100:00000001:0.0:1786176776.697322:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786176776.697322:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176776.697325:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176776.697328:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176776.697329:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176776.697331:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786176776.697334:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301676237
02000000:00000001:2.0:1786176776.697334:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176776.697337:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176776.697341:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
02000000:00000010:2.0:1786176776.697342:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786176776.697346:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176776.697351:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176776.697352:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786176776.697360:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176776.697363:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301676237
00000100:00000001:2.0:1786176776.697363:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176776.697370:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1786176776.697372:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134480.
00000100:00000001:2.0:1786176776.697379:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.697380:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00080000:2.0:1786176776.697382:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176776.697386:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176776.697387:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786176776.697388:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:2.0:1786176776.697389:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176776.697393:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786176776.697396:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176776.697397:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7000
00000100:00000001:0.0:1786176776.697402:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000400:00000200:2.0:1786176776.697403:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176776.697408:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176776.697410:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00100000:0.0:1786176776.697414:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176776/real 1786176776]  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176776.697415:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.697419:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176776.697421:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.697424:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176776.697429:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176776.697431:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176776.697434:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176776.697436:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176776.697439:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000001:0.0:1786176776.697440:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:2.0:1786176776.697440:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000001:0.0:1786176776.697444:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:2.0:1786176776.697445:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.697449:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1786176776.697450:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000100:00000040:2.0:1786176776.697451:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176776.697455:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786176776.697457:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000020:00000040:2.0:1786176776.697458:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000400:00000010:0.0:1786176776.697462:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00080000:2.0:1786176776.697464:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176776.697468:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1786176776.697469:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176776.697473:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:0.0:1786176776.697475:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176776.697477:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176776.697484:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176776.697489:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176776.697492:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000200:0.0:1786176776.697497:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1786176776.697498:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176776.697503:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176776.697529:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786176776.697532:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176776.697534:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176776.697535:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176776.697537:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176776.697538:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176776.697539:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176776.697543:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:0.0:1786176776.697547:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786176776.697549:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.697551:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.697556:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00100000:2.0:1786176776.697558:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176776.697560:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176776.697571:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176776.697572:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176776.697577:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176776.697586:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.697588:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176776.697600:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176776.697601:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176776.697608:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7010)
00000100:00000001:0.0:1786176776.697615:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176776.697620:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.697622:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176776.697624:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.697626:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176776.697628:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176776.697630:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176776.697633:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176776.697636:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176776.697639:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176776.697644:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856296192:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176776.697648:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.697649:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176776.697660:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf100 x1872934856296192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.697669:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176776.697671:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176776.697673:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176776.697675:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176776.697677:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176776.697678:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176776.697680:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176776.697681:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0800.
02000000:00000001:0.0:1786176776.697685:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176776.697687:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176776.697688:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf100.
00000100:00000001:0.0:1786176776.697690:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176776.697691:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.697693:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.697698:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176776.697699:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.697701:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176776.697702:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176776.697708:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176776.697710:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.697715:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176776.697716:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176776.697718:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176776.697719:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176776.697720:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176776.697722:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856296320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176776.697725:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176776.697726:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176776.697727:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176776.697729:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176776.697731:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176776.697733:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176776.697736:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176776.697738:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176776.697740:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176776.697743:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856296320, portal 10
00000100:00000001:0.0:1786176776.697744:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176776.697745:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000100:00000040:0.0:1786176776.697750:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.697755:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176776.697756:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176776.697758:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176776.697759:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856296320, offset 0
00000400:00000200:0.0:1786176776.697762:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176776.697765:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176776.697768:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176776.697770:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176776.697772:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176776.697774:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176776.697775:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176776.697777:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254486:254486:256:4294967295] 192.168.202.110@tcp LPNI seq info [254475:254475:8:4294967295]
00000400:00000200:0.0:1786176776.697780:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176776.697785:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176776.697788:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176776.697790:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176776.697793:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176776.697795:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176776.697799:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176776.697801:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1786176776.697804:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176776.697805:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176776.697807:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176776.697810:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176776.697812:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176776.697813:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176776.697815:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176776.697818:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.697823:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176776.697838:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.697846:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.697848:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176776.697851:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176776.697855:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176776.697859:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176776.697861:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301672458
00000100:00000001:0.0:1786176776.697863:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301676237
00000100:00000001:0.0:1786176776.697866:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301672458
00000100:00000001:0.0:1786176776.697868:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301676237
00000100:00000001:0.0:1786176776.697871:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176776.697872:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.697874:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176776.697877:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176776/real 1786176776]  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176776.697882:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176776.697884:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176776.697885:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176776.697886:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176776.697888:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176776.697889:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176776.697891:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176776.697897:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.697906:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176776.697907:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.697908:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176776.697909:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176776.697911:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.697915:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176776.697917:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.697922:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176776.697923:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.697924:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176776.697925:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.697926:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176776.697927:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176776.697929:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176776.697931:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176776.697933:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176776.697935:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176776.697940:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856296320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176776.697942:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176776.697944:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.697948:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176776.697949:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176776.697951:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176776.697952:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176776.697954:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176776.697955:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856296448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176776.697958:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176776.697959:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176776.697960:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176776.697961:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176776.697963:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176776.697965:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176776.697967:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176776.697968:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176776.697970:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176776.697972:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856296448, portal 10
00000100:00000001:0.0:1786176776.697974:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176776.697975:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178944 : -114548255372672 : ffff97d1a7edca80)
00000100:00000040:0.0:1786176776.697978:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.697983:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176776.697984:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176776.697985:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176776.697987:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856296448, offset 0
00000400:00000200:0.0:1786176776.697989:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176776.697992:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176776.697994:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176776.697996:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176776.697998:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176776.698000:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176776.698001:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176776.698004:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254487:254487:256:4294967295] 192.168.202.110@tcp LPNI seq info [254476:254476:8:4294967295]
00000400:00000200:0.0:1786176776.698006:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176776.698010:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176776.698014:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176776.698016:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176776.698018:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176776.698020:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176776.698023:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176776.698025:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1786176776.698027:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176776.698029:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176776.698031:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176776.698033:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176776.698035:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176776.698036:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176776.698038:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176776.698040:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698045:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176776.698047:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698051:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.698052:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176776.698054:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176776.698056:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176776.698058:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176776.698059:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301672583
00000100:00000001:0.0:1786176776.698060:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301676237
00000100:00000001:0.0:1786176776.698062:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301672583
00000100:00000001:0.0:1786176776.698063:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301676237
00000100:00000001:0.0:1786176776.698065:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176776.698066:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.698067:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176776.698069:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176776/real 1786176776]  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176776.698075:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176776.698076:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176776.698077:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176776.698078:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176776.698079:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176776.698080:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176776.698082:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176776.698088:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698095:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176776.698097:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698099:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176776.698100:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176776.698105:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698111:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176776.698116:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698124:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176776.698126:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698128:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176776.698129:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698131:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176776.698133:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176776.698136:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176776.698138:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176776.698140:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176776.698143:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176776.698148:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856296448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176776.698151:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176776.698153:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698157:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176776.698158:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176776.698159:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176776.698161:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176776.698162:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176776.698164:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856296576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176776.698166:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176776.698167:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176776.698168:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176776.698169:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176776.698171:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176776.698172:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176776.698175:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176776.698177:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176776.698179:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176776.698181:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856296576, portal 4
00000100:00000001:0.0:1786176776.698183:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176776.698184:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00000040:0.0:1786176776.698188:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698192:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176776.698194:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176776.698195:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176776.698197:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856296576, offset 0
00000400:00000200:0.0:1786176776.698200:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176776.698202:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176776.698205:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176776.698207:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176776.698209:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176776.698211:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176776.698212:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176776.698214:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254488:254488:256:4294967295] 192.168.202.110@tcp LPNI seq info [254477:254477:8:4294967295]
00000400:00000200:0.0:1786176776.698217:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176776.698221:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176776.698224:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176776.698226:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176776.698228:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176776.698230:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176776.698233:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176776.698234:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1786176776.698237:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176776.698238:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176776.698240:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176776.698242:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176776.698245:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176776.698246:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176776.698247:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176776.698250:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698255:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176776.698257:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698261:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.698262:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176776.698264:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176776.698266:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176776.698268:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176776.698270:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301672458
00000100:00000001:0.0:1786176776.698272:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301676237
00000100:00000001:0.0:1786176776.698273:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301672458
00000100:00000001:0.0:1786176776.698275:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301676237
00000100:00000001:0.0:1786176776.698276:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176776.698277:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.698278:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176776.698280:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176776/real 1786176776]  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176776.698285:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176776.698286:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176776.698287:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176776.698288:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176776.698289:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176776.698291:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176776.698293:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176776.698298:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698301:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176776.698302:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698303:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176776.698304:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176776.698306:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698310:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176776.698312:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698316:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176776.698319:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698320:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176776.698321:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698322:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176776.698323:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176776.698324:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176776.698326:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176776.698327:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176776.698330:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176776.698334:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856296576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176776.698336:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176776.698338:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698342:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176776.698343:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176776.698344:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176776.698345:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176776.698346:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176776.698348:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856296704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176776.698350:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176776.698351:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176776.698352:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176776.698353:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176776.698355:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176776.698356:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176776.698358:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176776.698360:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176776.698362:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176776.698363:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856296704, portal 4
00000100:00000001:0.0:1786176776.698365:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176776.698366:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000040:0.0:1786176776.698369:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698373:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176776.698375:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176776.698376:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176776.698377:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856296704, offset 0
00000400:00000200:0.0:1786176776.698379:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176776.698381:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176776.698383:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176776.698385:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176776.698387:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176776.698388:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176776.698390:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176776.698392:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254489:254489:256:4294967295] 192.168.202.110@tcp LPNI seq info [254478:254478:8:4294967295]
00000400:00000200:0.0:1786176776.698395:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176776.698398:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176776.698401:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176776.698402:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176776.698404:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176776.698406:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1786176776.698409:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176776.698410:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1786176776.698413:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176776.698414:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176776.698416:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176776.698418:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176776.698420:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176776.698421:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176776.698423:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176776.698425:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698434:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176776.698436:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698440:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.698441:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176776.698443:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176776.698444:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176776.698446:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176776.698447:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301672466
00000100:00000001:0.0:1786176776.698448:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301676237
00000100:00000001:0.0:1786176776.698450:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301672466
00000100:00000001:0.0:1786176776.698451:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301676237
00000100:00000001:0.0:1786176776.698453:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176776.698454:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.698455:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176776.698457:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176776/real 1786176776]  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176776.698462:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176776.698462:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176776.698463:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176776.698464:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176776.698465:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176776.698467:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176776.698468:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176776.698473:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698478:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176776.698479:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698480:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176776.698481:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176776.698483:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698488:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176776.698490:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698494:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176776.698495:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698496:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176776.698497:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698498:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176776.698499:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176776.698500:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176776.698502:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176776.698503:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176776.698505:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176776.698526:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856296704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176776.698528:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698530:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176776.698532:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856296320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698536:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176776.698537:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176776.698539:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176776.698541:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176776.698542:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176776.698543:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176776.698545:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176776.698546:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3400.
02000000:00000001:0.0:1786176776.698548:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176776.698549:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176776.698550:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
00000100:00000001:0.0:1786176776.698551:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176776.698552:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698553:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176776.698555:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edca80 x1872934856296448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698559:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176776.698560:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176776.698561:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176776.698562:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176776.698563:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176776.698564:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176776.698565:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176776.698566:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:0.0:1786176776.698567:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176776.698567:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176776.698568:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edca80.
00000100:00000001:0.0:1786176776.698569:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176776.698570:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698572:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176776.698573:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856296576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698578:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176776.698579:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176776.698579:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176776.698580:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176776.698581:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176776.698582:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176776.698583:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176776.698584:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:0.0:1786176776.698585:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176776.698586:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176776.698587:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:0.0:1786176776.698588:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176776.698589:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698590:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176776.698592:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856296704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176776.698596:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176776.698597:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176776.698597:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176776.698598:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176776.698599:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176776.698600:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176776.698601:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176776.698602:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:0.0:1786176776.698603:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176776.698603:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176776.698604:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:0.0:1786176776.698605:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176776.698606:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698607:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176776.698610:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176776.698610:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176776.698612:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176776.698613:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176777.272733:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176777.272738:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176777.272742:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176779.320722:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176779.320726:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176779.320728:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176780.344696:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176780.344700:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176780.344703:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176780.536617:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301676237, at 4301676288
00000100:00000010:0.0:1786176780.536634:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176781.816923:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176781.816929:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176781.816932:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176781.816941:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856296832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.816958:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176781.816963:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176781.816967:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176781.816970:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176781.816973:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176781.816980:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856296832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176781.816985:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176781.816989:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176781.816993:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176781.816996:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176781.817002:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176781.817007:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176781.817014:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176781.817018:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176781.817026:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176781.817035:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856296832, portal 25
00000100:00000001:0.0:1786176781.817039:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176781.817041:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000100:00000040:0.0:1786176781.817047:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856296832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.817057:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176781.817062:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176781.817066:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176781.817071:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856296832, offset 0
00000400:00000200:0.0:1786176781.817080:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176781.817091:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176781.817097:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176781.817106:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176781.817113:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176781.817122:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176781.817129:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176781.817137:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254490:254490:256:4294967295] 192.168.202.110@tcp LPNI seq info [254479:254479:8:4294967295]
00000400:00000200:0.0:1786176781.817149:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176781.817166:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176781.817175:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176781.817183:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176781.817191:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176781.817196:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176781.817206:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176781.817210:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786176781.817216:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176781.817219:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176781.817225:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176781.817230:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176781.817235:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176781.817238:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176781.817245:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176781.817251:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856296832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.817266:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176781.817272:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856296832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.817294:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.817298:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176781.817306:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176781.817312:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176781.817316:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301681357
00000100:00000010:0.0:1786176781.817323:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176781.817332:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176781.817339:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301681357
00000100:00000010:0.0:1786176781.817344:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1851341e0.
00000100:00000001:0.0:1786176781.817349:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176781.817355:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176781.817361:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.817365:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176781.817371:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176781/real 1786176781]  req@ffff97d1ba69d500 x1872934856296832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176781.817385:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176781.817389:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176781.817392:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176781.817396:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176781.817399:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176781.817402:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176781.817407:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856296832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176781.817418:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856296832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.817425:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176781.817427:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.817432:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176781.817434:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176781.817442:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856296832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.817459:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176781.817466:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856296832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.817474:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176781.817478:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.817480:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176781.817481:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.817485:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176781.817487:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176781.817493:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176781.817500:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176781.817506:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176781.817534:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856296832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176781.817542:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856296832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176781.817550:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.817554:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176781.817559:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856296832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.817571:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176781.817575:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176781.817579:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176781.817583:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176781.817585:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176781.817588:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176781.817591:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176781.817594:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786176781.817599:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176781.817602:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176781.817604:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:0.0:1786176781.817608:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176781.817611:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.817614:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.817621:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176781.817624:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.817629:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176781.817631:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176781.817634:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176781.817640:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856296960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.817653:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176781.817655:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176781.817659:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176781.817662:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176781.817665:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176781.817672:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856296960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176781.817678:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176781.817680:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176781.817682:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176781.817686:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176781.817689:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176781.817692:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176781.817697:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176781.817702:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176781.817707:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176781.817710:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856296960, portal 10
00000100:00000001:0.0:1786176781.817714:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176781.817717:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:0.0:1786176781.817725:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856296960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.817736:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176781.817741:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176781.817744:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176781.817748:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856296960, offset 0
00000400:00000200:0.0:1786176781.817754:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176781.817758:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176781.817764:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176781.817768:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176781.817773:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176781.817791:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176781.817795:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176781.817800:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254491:254491:256:4294967295] 192.168.202.110@tcp LPNI seq info [254480:254480:8:4294967295]
00000400:00000200:0.0:1786176781.817805:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176781.817813:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176781.817819:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176781.817825:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176781.817832:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176781.817836:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176781.817844:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176781.817849:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786176781.817855:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176781.817859:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176781.817863:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176781.817869:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176781.817874:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176781.817876:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176781.817880:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176781.817886:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856296960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.817897:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176781.817901:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856296960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.817911:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.817914:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176781.817918:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176781.817924:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176781.817931:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176781.817935:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301677578
00000100:00000001:0.0:1786176781.817938:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301681357
00000100:00000001:0.0:1786176781.817941:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301677578
00000100:00000001:0.0:1786176781.817945:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301681357
00000100:00000001:0.0:1786176781.817951:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176781.817955:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.817960:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176781.817967:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176781/real 1786176781]  req@ffff97d1ba69f100 x1872934856296960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176781.817987:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176781.817990:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176781.817993:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176781.817996:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176781.818000:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176781.818004:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176781.818008:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856296960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176781.818022:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856296960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818032:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176781.818034:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818037:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176781.818039:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176781.818046:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856296960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818056:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176781.818063:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856296960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818071:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176781.818074:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818078:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176781.818080:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818084:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176781.818088:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176781.818090:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176781.818097:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176781.818101:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176781.818108:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856296960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176781.818127:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856296960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176781.818136:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176781.818143:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856297088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818157:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176781.818161:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176781.818164:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176781.818167:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176781.818171:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176781.818179:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176781.818187:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176781.818191:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176781.818196:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176781.818201:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176781.818206:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176781.818211:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176781.818218:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176781.818223:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176781.818229:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176781.818232:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856297088, portal 10
00000100:00000001:0.0:1786176781.818235:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176781.818237:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:0.0:1786176781.818245:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856297088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818252:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176781.818254:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176781.818256:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176781.818257:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856297088, offset 0
00000400:00000200:0.0:1786176781.818261:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176781.818264:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176781.818266:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176781.818269:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176781.818270:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176781.818272:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176781.818274:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176781.818276:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254492:254492:256:4294967295] 192.168.202.110@tcp LPNI seq info [254481:254481:8:4294967295]
00000400:00000200:0.0:1786176781.818279:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176781.818283:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176781.818285:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176781.818288:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176781.818290:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176781.818292:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176781.818295:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176781.818297:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786176781.818299:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176781.818301:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176781.818303:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176781.818305:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176781.818307:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176781.818308:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176781.818310:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176781.818312:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856297088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818318:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176781.818319:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856297088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818324:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.818325:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176781.818327:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176781.818329:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176781.818331:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176781.818333:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301677703
00000100:00000001:0.0:1786176781.818335:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301681357
00000100:00000001:0.0:1786176781.818336:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301677703
00000100:00000001:0.0:1786176781.818338:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301681357
00000100:00000001:0.0:1786176781.818339:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176781.818340:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.818341:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176781.818344:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176781/real 1786176781]  req@ffff97d1ba69f480 x1872934856297088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176781.818348:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176781.818349:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176781.818350:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176781.818352:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176781.818353:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176781.818354:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176781.818356:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856297088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176781.818361:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856297088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818365:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176781.818366:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818367:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176781.818368:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176781.818370:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856297088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818374:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176781.818376:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856297088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818381:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176781.818383:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818384:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176781.818385:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818386:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176781.818387:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176781.818389:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176781.818391:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176781.818393:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176781.818395:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856297088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176781.818399:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176781.818402:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818403:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176781.818405:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856296960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818409:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176781.818410:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176781.818412:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176781.818413:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176781.818414:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176781.818415:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176781.818417:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176781.818418:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786176781.818421:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176781.818421:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176781.818423:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:0.0:1786176781.818425:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176781.818426:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818427:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176781.818429:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856297088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818437:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176781.818438:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176781.818439:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176781.818440:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176781.818441:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176781.818441:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176781.818443:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176781.818443:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786176781.818444:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176781.818445:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176781.818446:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:0.0:1786176781.818447:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176781.818448:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818449:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818451:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176781.818452:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.818453:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176781.818454:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176781.818455:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176781.818457:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856297216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818461:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176781.818462:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176781.818463:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176781.818464:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176781.818466:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176781.818467:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176781.818470:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176781.818471:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176781.818472:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176781.818473:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176781.818475:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176781.818476:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176781.818479:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176781.818480:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176781.818483:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176781.818485:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856297216, portal 4
00000100:00000001:0.0:1786176781.818486:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176781.818487:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:0.0:1786176781.818492:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856297216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818496:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176781.818498:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176781.818499:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176781.818500:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856297216, offset 0
00000400:00000200:0.0:1786176781.818503:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176781.818505:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176781.818507:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176781.818524:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176781.818526:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176781.818528:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176781.818530:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176781.818532:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254493:254493:256:4294967295] 192.168.202.110@tcp LPNI seq info [254482:254482:8:4294967295]
00000400:00000200:0.0:1786176781.818535:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176781.818539:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176781.818542:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176781.818544:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176781.818547:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176781.818549:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176781.818552:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176781.818554:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786176781.818556:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176781.818557:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176781.818559:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176781.818562:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176781.818564:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176781.818565:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176781.818567:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176781.818569:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856297216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818575:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176781.818577:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856297216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818581:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.818582:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176781.818584:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176781.818586:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176781.818588:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176781.818589:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301677579
00000100:00000001:0.0:1786176781.818591:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301681357
00000100:00000001:0.0:1786176781.818592:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301677579
00000100:00000001:0.0:1786176781.818594:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301681357
00000100:00000001:0.0:1786176781.818595:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176781.818596:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.818597:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176781.818599:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176781/real 1786176781]  req@ffff97d1ba69ca80 x1872934856297216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176781.818604:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176781.818605:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176781.818606:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176781.818608:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176781.818609:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176781.818610:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176781.818612:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856297216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176781.818617:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856297216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818622:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176781.818623:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818624:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176781.818625:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176781.818627:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856297216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818631:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176781.818634:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856297216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818638:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176781.818639:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818640:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176781.818641:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818642:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176781.818643:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176781.818644:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176781.818646:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176781.818647:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176781.818650:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856297216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176781.818654:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176781.818656:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176781.818658:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856297344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818663:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176781.818664:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176781.818665:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176781.818666:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176781.818668:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176781.818670:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297344:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176781.818672:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176781.818673:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176781.818674:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176781.818675:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176781.818676:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176781.818677:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176781.818679:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176781.818681:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176781.818683:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176781.818685:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856297344, portal 4
00000100:00000001:0.0:1786176781.818686:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176781.818687:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000040:0.0:1786176781.818691:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856297344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818695:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176781.818697:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176781.818698:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176781.818700:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856297344, offset 0
00000400:00000200:0.0:1786176781.818702:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176781.818704:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176781.818706:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176781.818708:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176781.818709:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176781.818710:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176781.818712:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176781.818714:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254494:254494:256:4294967295] 192.168.202.110@tcp LPNI seq info [254483:254483:8:4294967295]
00000400:00000200:0.0:1786176781.818717:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176781.818720:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176781.818724:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176781.818726:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176781.818728:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176781.818730:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786176781.818732:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176781.818734:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786176781.818737:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176781.818738:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176781.818740:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176781.818742:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176781.818744:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176781.818745:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176781.818747:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176781.818749:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856297344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818753:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176781.818755:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856297344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818759:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.818760:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176781.818762:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176781.818763:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176781.818766:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176781.818767:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301677587
00000100:00000001:0.0:1786176781.818768:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301681357
00000100:00000001:0.0:1786176781.818770:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301677587
00000100:00000001:0.0:1786176781.818771:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301681357
00000100:00000001:0.0:1786176781.818772:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176781.818773:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.818791:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176781.818795:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176781/real 1786176781]  req@ffff97d1ba69fb80 x1872934856297344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176781.818801:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176781.818802:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176781.818803:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176781.818805:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176781.818806:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176781.818808:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176781.818810:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856297344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176781.818815:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856297344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818819:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176781.818820:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818821:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176781.818822:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176781.818824:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856297344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818828:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176781.818830:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856297344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818835:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176781.818836:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818837:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176781.818838:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818840:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176781.818841:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176781.818842:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176781.818844:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176781.818845:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176781.818848:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856297344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176781.818853:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297344:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176781.818855:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818857:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176781.818859:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856297216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818862:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176781.818863:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176781.818865:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176781.818866:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176781.818867:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176781.818868:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176781.818869:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176781.818871:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786176781.818872:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176781.818873:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176781.818874:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:0.0:1786176781.818876:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176781.818877:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818878:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176781.818880:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856297344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176781.818883:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176781.818884:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176781.818885:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176781.818886:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176781.818887:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176781.818888:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176781.818889:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176781.818889:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786176781.818891:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176781.818892:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176781.818893:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:0.0:1786176781.818894:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176781.818895:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818896:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176781.818898:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176781.818899:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176781.818901:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176781.818902:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176782.392667:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176782.392671:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176782.392673:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176783.416700:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176783.416704:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176783.416709:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176785.464680:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176785.464684:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176785.464687:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176785.656603:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301681357, at 4301681408
00000100:00000010:0.0:1786176785.656623:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000020:00000001:0.0:1786176786.488744:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176786.488748:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176786.488751:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176786.936619:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176786.936628:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176786.936631:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176786.936637:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176786.936643:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176786.936649:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7010
00000400:00000200:2.0:1786176786.936659:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176786.936666:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176786.936671:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176786.936677:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.936680:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176786.936682:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.936685:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176786.936687:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176786.936688:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176786.936690:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176786.936693:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176786.936695:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176786.936697:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176786.936699:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.936701:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176786.936702:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.936703:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176786.936705:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176786.936708:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176786.936709:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176786.936712:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176786.936714:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176786.936716:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.936717:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176786.936719:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176786.936738:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176786.936741:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176786.936745:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176786.936748:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.936751:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176786.936754:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176786.936759:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176786.936767:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786176786.936774:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176786.936783:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176786.936795:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176786.936800:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176786.936806:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176786.936829:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176786.936833:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176786.936836:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176786.936838:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176786.936839:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176786.936842:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176786.936844:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7010
00000400:00000200:2.0:1786176786.936847:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176786.936850:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176786.936851:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176786.936853:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.936855:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176786.936857:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.936859:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176786.936860:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176786.936862:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176786.936863:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176786.936865:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176786.936867:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176786.936868:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176786.936871:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.936873:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176786.936875:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.936876:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176786.936877:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176786.936880:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176786.936881:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786176786.936882:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176786.936885:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786176786.936887:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1786176786.936890:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176786.936892:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:2.0:1786176786.936894:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176786.936902:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176786.936903:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176786.936907:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176786.936912:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.936916:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176786.936920:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000100:00000001:0.0:1786176786.936923:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1786176786.936924:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:0.0:1786176786.936928:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176786.936928:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176786.936930:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.936933:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176786.936936:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786176786.936936:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176786.936938:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176786.936941:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000010:2.0:1786176786.936942:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786176786.936948:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176786.936949:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176786.936957:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856297472:192.168.202.110@tcp:250:kworker.0
00000100:00100000:2.0:1786176786.936958:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176786.936968:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1786176786.936970:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176786.936973:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176786.936975:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:2.0:1786176786.936976:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786176786.936980:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176786.936986:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786176786.936993:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176786.936996:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
00000100:00080000:2.0:1786176786.936996:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176786.936999:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176786.937000:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176786.937001:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176786.937002:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176786.937004:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7010
00000400:00000200:2.0:1786176786.937007:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176786.937009:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:0.0:1786176786.937012:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176786.937013:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176786.937018:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.937021:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176786.937024:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786176786.937027:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176786.937027:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176786.937031:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176786.937033:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176786.937034:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000010:0.0:1786176786.937037:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000020:00000040:2.0:1786176786.937037:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176786.937041:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176786.937044:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176786.937046:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.937047:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1786176786.937049:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000040:2.0:1786176786.937049:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.937055:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176786.937056:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176786.937058:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:0.0:1786176786.937060:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856297472, portal 25
00000100:00000001:2.0:1786176786.937060:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176786.937065:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786176786.937067:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
02000000:00000001:2.0:1786176786.937068:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176786.937070:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
02000000:00000001:2.0:1786176786.937072:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.937075:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176786.937078:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176786.937080:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.937083:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176786.937085:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786176786.937087:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786176786.937091:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176786.937094:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176786.937096:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.937100:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176786.937102:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176786.937104:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176786.937107:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786176786.937110:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176786.937113:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176786.937118:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786176786.937122:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176786.937124:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176786.937127:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176786.937133:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176786.937140:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176786.937145:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856297472, offset 0
00000100:00000001:2.0:1786176786.937147:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176786.937150:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176786.937153:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1786176786.937156:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176786.937156:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176786.937159:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176786.937163:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176786.937165:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7010
00000400:00000200:0.0:1786176786.937172:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176786.937172:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176786.937175:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176786.937178:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176786.937181:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786176786.937185:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1786176786.937185:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176786.937188:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786176786.937196:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786176786.937197:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176786.937200:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176786.937201:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176786.937202:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:0.0:1786176786.937205:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:2.0:1786176786.937206:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176786.937210:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:0.0:1786176786.937213:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:2.0:1786176786.937213:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176786.937218:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.937220:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786176786.937222:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:2.0:1786176786.937225:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.937228:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176786.937232:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000400:00000200:0.0:1786176786.937233:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254495:254495:256:4294967295] 192.168.202.110@tcp LPNI seq info [254484:254484:8:4294967295]
00000100:00080000:2.0:1786176786.937236:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176786.937242:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176786.937245:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786176786.937246:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786176786.937247:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176786.937251:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.937252:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176786.937254:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176786.937256:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.937260:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176786.937263:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000400:00000200:0.0:1786176786.937265:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:2.0:1786176786.937267:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176786.937272:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176786.937273:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.937276:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:0.0:1786176786.937278:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1786176786.937279:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176786.937281:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176786.937285:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000800:00000010:0.0:1786176786.937288:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000100:00000001:2.0:1786176786.937288:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176786.937293:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176786.937301:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000200:0.0:1786176786.937302:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176786.937303:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176786.937309:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1786176786.937311:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1786176786.937317:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176786.937320:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1786176786.937324:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786176786.937330:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:0.1:1786176786.937330:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1786176786.937333:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176786.937335:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176786.937338:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000800:00000100:0.0:1786176786.937341:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176786.937343:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7010
00000800:00000010:0.0:1786176786.937348:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:2.0:1786176786.937348:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176786.937353:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176786.937355:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176786.937358:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786176786.937359:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1786176786.937360:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176786.937364:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.937366:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176786.937369:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176786.937371:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000400:00000200:0.0:1786176786.937373:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:2.0:1786176786.937373:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176786.937376:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176786.937380:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176786.937382:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:0.0:1786176786.937383:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000100:00000040:2.0:1786176786.937385:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.937388:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1786176786.937389:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000040:2.0:1786176786.937391:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176786.937395:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176786.937397:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:0.0:1786176786.937398:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:2.0:1786176786.937400:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176786.937404:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176786.937405:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176786.937407:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:0.0:1786176786.937409:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176786.937409:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.937410:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176786.937413:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176786.937417:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.937420:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176786.937425:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786176786.937429:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176786.937433:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176786.937434:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.937435:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786176786.937436:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1786176786.937437:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176786.937441:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176786.937445:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1786176786.937445:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786176786.937449:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176786.937456:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176786.937466:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.937467:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176786.937471:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786176786.937471:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176786.937475:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.937481:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176786.937481:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176786.937489:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7020)
00000400:00000200:0.0:1786176786.937493:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176786.937498:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301686477
00000100:00000010:0.0:1786176786.937504:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176786.937536:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176786.937543:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301686478
00000100:00000010:0.0:1786176786.937547:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134ea0.
00000100:00000001:0.0:1786176786.937551:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176786.937555:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176786.937558:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176786.937561:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176786.937566:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176786/real 1786176786]  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176786.937576:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176786.937578:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176786.937580:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176786.937583:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176786.937585:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176786.937588:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176786.937592:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176786.937600:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.937621:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176786.937623:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.937625:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176786.937627:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176786.937631:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.937638:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176786.937643:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.937651:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176786.937653:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.937655:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176786.937656:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.937659:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176786.937661:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176786.937664:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176786.937668:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176786.937671:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176786.937675:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176786.937684:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856297472:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176786.937688:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.937690:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176786.937693:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856297472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.937701:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176786.937704:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176786.937707:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176786.937710:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176786.937712:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176786.937714:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176786.937717:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176786.937719:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:0.0:1786176786.937723:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176786.937737:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176786.937740:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:0.0:1786176786.937743:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176786.937745:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.937747:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.937755:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176786.937756:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176786.937758:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176786.937760:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176786.937761:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176786.937765:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.937773:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176786.937775:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176786.937777:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176786.937779:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176786.937782:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176786.937785:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297600:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176786.937788:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176786.937790:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176786.937792:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176786.937794:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176786.937797:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176786.937799:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176786.937802:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176786.937805:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176786.937809:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176786.937812:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856297600, portal 10
00000100:00000001:0.0:1786176786.937814:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176786.937816:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:0.0:1786176786.937823:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.937830:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176786.937832:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176786.937835:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176786.937837:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856297600, offset 0
00000400:00000200:0.0:1786176786.937841:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176786.937845:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176786.937849:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176786.937852:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176786.937855:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176786.937858:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176786.937861:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176786.937864:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254496:254496:256:4294967295] 192.168.202.110@tcp LPNI seq info [254485:254485:8:4294967295]
00000400:00000200:0.0:1786176786.937868:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176786.937875:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176786.937880:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176786.937883:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176786.937887:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176786.937890:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176786.937895:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176786.937898:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786176786.937902:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176786.937904:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176786.937907:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176786.937911:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176786.937914:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176786.937916:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176786.937918:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176786.937922:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.937937:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176786.937940:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.937947:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176786.937949:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176786.937952:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176786.937956:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176786.937960:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176786.937962:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301682698
00000100:00000001:0.0:1786176786.937965:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301686477
00000100:00000001:0.0:1786176786.937967:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301682698
00000100:00000001:0.0:1786176786.937969:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301686478
00000100:00000001:0.0:1786176786.937972:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176786.937974:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176786.937976:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176786.937980:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176786/real 1786176786]  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176786.937989:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176786.937990:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176786.937992:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176786.937994:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176786.937996:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176786.937998:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176786.938001:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176786.938010:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938016:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176786.938018:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938020:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176786.938021:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176786.938025:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938032:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176786.938036:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938044:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176786.938046:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938048:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176786.938050:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938051:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176786.938053:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176786.938056:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176786.938059:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176786.938062:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176786.938066:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176786.938074:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297600:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176786.938079:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176786.938082:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938090:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176786.938091:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176786.938093:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176786.938095:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176786.938097:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176786.938101:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176786.938104:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176786.938105:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176786.938107:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176786.938109:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176786.938112:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176786.938114:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176786.938118:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176786.938120:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176786.938124:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176786.938127:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856297728, portal 10
00000100:00000001:0.0:1786176786.938129:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176786.938130:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:0.0:1786176786.938136:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938142:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176786.938144:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176786.938146:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176786.938148:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856297728, offset 0
00000400:00000200:0.0:1786176786.938153:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176786.938156:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176786.938160:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176786.938163:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176786.938165:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176786.938168:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176786.938170:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176786.938173:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254497:254497:256:4294967295] 192.168.202.110@tcp LPNI seq info [254486:254486:8:4294967295]
00000400:00000200:0.0:1786176786.938177:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176786.938183:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176786.938188:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176786.938191:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176786.938195:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176786.938198:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176786.938202:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176786.938204:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786176786.938208:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176786.938210:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176786.938213:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176786.938217:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176786.938220:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176786.938222:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176786.938224:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176786.938228:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938235:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176786.938238:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938245:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176786.938247:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176786.938249:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176786.938252:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176786.938255:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176786.938257:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301682823
00000100:00000001:0.0:1786176786.938260:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301686477
00000100:00000001:0.0:1786176786.938262:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301682823
00000100:00000001:0.0:1786176786.938264:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301686478
00000100:00000001:0.0:1786176786.938266:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176786.938268:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176786.938270:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176786.938273:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176786/real 1786176786]  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176786.938280:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176786.938281:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176786.938283:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176786.938285:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176786.938287:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176786.938289:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176786.938292:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176786.938301:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938308:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176786.938310:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938311:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176786.938313:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176786.938317:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938324:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176786.938329:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938336:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176786.938339:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938341:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176786.938342:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938344:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176786.938347:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176786.938349:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176786.938353:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176786.938356:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176786.938361:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176786.938375:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176786.938379:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176786.938382:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938387:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176786.938388:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176786.938389:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176786.938391:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176786.938392:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176786.938394:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176786.938396:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176786.938397:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176786.938398:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176786.938399:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176786.938402:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176786.938403:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176786.938406:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176786.938408:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176786.938410:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176786.938412:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856297856, portal 4
00000100:00000001:0.0:1786176786.938414:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176786.938414:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000040:0.0:1786176786.938419:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938423:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176786.938425:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176786.938426:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176786.938427:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856297856, offset 0
00000400:00000200:0.0:1786176786.938430:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176786.938433:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176786.938435:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176786.938437:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176786.938439:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176786.938441:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176786.938442:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176786.938444:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254498:254498:256:4294967295] 192.168.202.110@tcp LPNI seq info [254487:254487:8:4294967295]
00000400:00000200:0.0:1786176786.938447:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176786.938452:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176786.938455:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176786.938457:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176786.938459:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176786.938461:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176786.938464:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176786.938465:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786176786.938468:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176786.938470:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176786.938472:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176786.938474:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176786.938476:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176786.938477:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176786.938479:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176786.938481:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938486:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176786.938488:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938492:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176786.938493:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176786.938495:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176786.938497:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176786.938498:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176786.938500:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301682698
00000100:00000001:0.0:1786176786.938502:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301686477
00000100:00000001:0.0:1786176786.938504:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301682698
00000100:00000001:0.0:1786176786.938505:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301686478
00000100:00000001:0.0:1786176786.938506:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176786.938507:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176786.938508:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176786.938526:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176786/real 1786176786]  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176786.938532:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176786.938533:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176786.938534:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176786.938535:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176786.938537:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176786.938538:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176786.938540:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176786.938546:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938550:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176786.938550:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938551:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176786.938552:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176786.938554:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938558:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176786.938561:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938565:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176786.938567:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938568:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176786.938569:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938570:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176786.938572:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176786.938573:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176786.938575:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176786.938576:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176786.938579:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176786.938583:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176786.938586:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176786.938588:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938592:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176786.938593:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176786.938595:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176786.938596:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176786.938597:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176786.938599:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176786.938600:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176786.938601:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176786.938602:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176786.938604:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176786.938606:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176786.938607:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176786.938609:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176786.938611:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176786.938612:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176786.938614:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856297984, portal 4
00000100:00000001:0.0:1786176786.938616:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176786.938617:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:0.0:1786176786.938620:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938625:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176786.938626:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176786.938628:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176786.938629:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856297984, offset 0
00000400:00000200:0.0:1786176786.938632:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176786.938634:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176786.938636:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176786.938638:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176786.938639:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176786.938641:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176786.938642:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176786.938645:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254499:254499:256:4294967295] 192.168.202.110@tcp LPNI seq info [254488:254488:8:4294967295]
00000400:00000200:0.0:1786176786.938647:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176786.938651:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176786.938654:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176786.938655:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176786.938658:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176786.938659:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786176786.938662:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176786.938664:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786176786.938666:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176786.938668:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176786.938669:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176786.938671:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176786.938673:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176786.938674:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176786.938676:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176786.938678:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938687:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176786.938689:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938693:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176786.938694:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176786.938696:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176786.938698:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176786.938699:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176786.938701:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301682707
00000100:00000001:0.0:1786176786.938702:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301686477
00000100:00000001:0.0:1786176786.938704:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301682707
00000100:00000001:0.0:1786176786.938705:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301686478
00000100:00000001:0.0:1786176786.938707:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176786.938708:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176786.938709:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176786.938711:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176786/real 1786176786]  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176786.938715:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176786.938716:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176786.938717:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176786.938718:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176786.938719:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176786.938720:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176786.938722:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176786.938743:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938748:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176786.938749:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938751:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176786.938752:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176786.938754:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938759:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176786.938761:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938766:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176786.938767:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938768:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176786.938769:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938770:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176786.938771:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176786.938772:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176786.938774:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176786.938775:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176786.938778:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176786.938782:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856297984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176786.938785:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938786:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176786.938788:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856297600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938792:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176786.938793:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176786.938795:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176786.938796:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176786.938797:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176786.938799:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176786.938800:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176786.938801:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:0.0:1786176786.938803:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176786.938804:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176786.938805:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:0.0:1786176786.938806:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176786.938807:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938808:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176786.938810:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856297728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938815:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176786.938816:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176786.938816:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176786.938818:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176786.938818:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176786.938819:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176786.938820:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176786.938821:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:0.0:1786176786.938823:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176786.938824:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176786.938825:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:0.0:1786176786.938826:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176786.938827:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938828:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176786.938829:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856297856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938833:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176786.938834:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176786.938835:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176786.938836:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176786.938836:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176786.938837:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176786.938838:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176786.938839:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:0.0:1786176786.938840:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176786.938841:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176786.938842:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:0.0:1786176786.938844:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176786.938844:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938845:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176786.938847:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856297984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176786.938851:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176786.938852:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176786.938853:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176786.938854:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176786.938854:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176786.938855:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176786.938856:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176786.938857:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:0.0:1786176786.938858:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176786.938859:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176786.938860:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:0.0:1786176786.938861:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176786.938862:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938863:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176786.938865:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176786.938866:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176786.938867:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176786.938868:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176788.536744:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176788.536747:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176788.536750:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176789.560691:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176789.560695:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176789.560698:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176790.776591:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301686477, at 4301686528
00000100:00000010:0.0:1786176790.776608:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000020:00000001:2.0:1786176791.608703:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176791.608706:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176791.608709:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176792.056591:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176792.056609:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176792.056614:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176792.056619:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176792.056626:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176792.056636:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7015
00000400:00000200:2.0:1786176792.056649:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176792.056658:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176792.056660:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176792.056663:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.056666:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176792.056668:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.056671:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176792.056674:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176792.056687:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176792.056690:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176792.056694:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176792.056696:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176792.056698:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176792.056701:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.056703:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176792.056705:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.056706:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176792.056708:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176792.056711:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176792.056713:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176792.056716:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176792.056719:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176792.056721:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176792.056724:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176792.056726:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176792.056734:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786176792.056736:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176792.056740:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176792.056741:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176792.056744:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176792.056746:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176792.056750:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176792.056757:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786176792.056762:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176792.056769:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176792.056777:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176792.056781:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176792.056784:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176792.056801:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176792.056803:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176792.056806:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176792.056807:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176792.056808:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176792.056811:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176792.056813:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7015
00000400:00000200:2.0:1786176792.056817:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176792.056820:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176792.056823:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176792.056825:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.056828:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176792.056831:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.056834:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176792.056837:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176792.056838:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176792.056840:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176792.056843:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176792.056845:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176792.056847:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786176792.056848:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1786176792.056851:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176792.056854:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176792.056856:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786176792.056859:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786176792.056860:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.056864:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176792.056868:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000040:0.0:1786176792.056871:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176792.056875:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176792.056886:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176792.056889:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.056890:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176792.056895:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176792.056897:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176792.056901:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176792.056903:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176792.056905:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176792.056907:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176792.056910:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00100000:0.0:1786176792.056913:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856298112:192.168.202.110@tcp:250:kworker.0
00000008:00100000:2.0:1786176792.056916:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1786176792.056922:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1786176792.056923:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176792.056927:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176792.056928:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:2.0:1786176792.056932:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
02000000:00000001:0.0:1786176792.056933:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786176792.056939:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786176792.056940:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1786176792.056945:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176792.056947:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176792.056948:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
00000100:00000001:2.0:1786176792.056950:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176792.056953:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176792.056954:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176792.056959:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176792.056965:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:2.0:1786176792.056966:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:2.0:1786176792.056971:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786176792.056972:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000100:00100000:2.0:1786176792.056977:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1786176792.056981:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000001:2.0:1786176792.056988:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176792.056990:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786176792.056993:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856298112, portal 25
00000100:00000040:2.0:1786176792.056994:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.057000:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176792.057004:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454181632 : -114548255369984 : ffff97d1a7edd500)
00000100:00000001:2.0:1786176792.057005:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176792.057010:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176792.057014:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176792.057016:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:0.0:1786176792.057018:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176792.057018:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176792.057023:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176792.057028:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7015
00000400:00000200:2.0:1786176792.057043:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176792.057049:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176792.057052:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176792.057056:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176792.057057:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786176792.057058:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176792.057063:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.057065:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:0.0:1786176792.057068:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000001:2.0:1786176792.057069:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176792.057071:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176792.057074:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000010:0.0:1786176792.057076:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000020:00000040:2.0:1786176792.057077:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176792.057083:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000200:0.0:1786176792.057084:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856298112, offset 0
00000020:00000001:2.0:1786176792.057086:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176792.057094:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786176792.057098:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176792.057099:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176792.057106:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.057109:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786176792.057112:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786176792.057115:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176792.057121:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1786176792.057125:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1786176792.057127:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176792.057131:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786176792.057134:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:2.0:1786176792.057136:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176792.057139:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176792.057141:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:0.0:1786176792.057143:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00100000:2.0:1786176792.057144:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176792.057148:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176792.057150:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176792.057151:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000010:2.0:1786176792.057154:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000400:00000200:0.0:1786176792.057159:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000040:2.0:1786176792.057159:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176792.057165:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176792.057167:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176792.057171:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254500:254500:256:4294967295] 192.168.202.110@tcp LPNI seq info [254489:254489:8:4294967295]
00000100:00000001:2.0:1786176792.057171:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176792.057174:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176792.057176:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176792.057183:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:2.0:1786176792.057183:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786176792.057187:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176792.057194:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1786176792.057205:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786176792.057209:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176792.057213:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176792.057219:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1786176792.057220:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176792.057230:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1786176792.057233:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000100:00080000:2.0:1786176792.057233:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176792.057239:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176792.057240:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176792.057241:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176792.057245:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:0.0:1786176792.057246:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786176792.057252:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7015
00000800:00000200:0.0:1786176792.057254:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000400:00000200:2.0:1786176792.057260:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:0.1:1786176792.057266:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:2.0:1786176792.057268:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176792.057274:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:0.1:1786176792.057274:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:2.0:1786176792.057282:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000100:0.0:1786176792.057287:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176792.057288:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:0.0:1786176792.057292:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000100:00000040:2.0:1786176792.057294:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.057298:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176792.057300:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:0.0:1786176792.057301:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:2.0:1786176792.057302:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176792.057306:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176792.057309:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176792.057313:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:0.0:1786176792.057314:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:2.0:1786176792.057316:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176792.057322:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786176792.057324:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000100:00000001:2.0:1786176792.057325:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1786176792.057329:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000040:2.0:1786176792.057330:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.057336:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786176792.057337:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:2.0:1786176792.057340:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000200:0.0:1786176792.057346:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176792.057347:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176792.057354:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176792.057358:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176792.057363:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176792.057367:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1786176792.057367:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176792.057371:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176792.057374:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000040:0.0:1786176792.057376:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1786176792.057377:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176792.057380:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176792.057385:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786176792.057389:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176792.057394:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1786176792.057396:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176792.057397:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.057403:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786176792.057403:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176792.057407:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176792.057409:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176792.057412:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:0.0:1786176792.057413:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176792.057415:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176792.057421:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1786176792.057423:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176792.057428:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301691597
00000100:00000001:2.0:1786176792.057431:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176792.057434:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176792.057437:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000040:2.0:1786176792.057438:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176792.057447:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.057448:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:2.0:1786176792.057449:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176792.057454:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176792.057455:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176792.057457:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176792.057458:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301691597
00000100:00000001:2.0:1786176792.057459:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176792.057463:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7015
00000100:00000010:0.0:1786176792.057466:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134600.
00000400:00000200:2.0:1786176792.057469:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176792.057473:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000400:00000200:2.0:1786176792.057474:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176792.057478:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786176792.057481:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1786176792.057482:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.057486:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786176792.057488:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176792.057489:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176792.057493:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1786176792.057493:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176792.057498:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176792.057499:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176792.057501:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176792.057503:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00100000:0.0:1786176792.057505:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176792/real 1786176792]  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786176792.057505:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176792.057536:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176792.057542:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.057545:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176792.057549:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176792.057553:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176792.057557:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786176792.057562:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176792.057564:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1786176792.057565:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:2.0:1786176792.057566:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000001:0.0:1786176792.057570:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:2.0:1786176792.057571:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176792.057575:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786176792.057576:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000100:00000001:2.0:1786176792.057577:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176792.057579:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176792.057582:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176792.057583:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000100:00000001:2.0:1786176792.057584:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786176792.057589:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000010:2.0:1786176792.057590:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000100:00000001:0.0:1786176792.057595:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:2.0:1786176792.057595:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176792.057598:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176792.057602:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786176792.057604:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176792.057606:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176792.057609:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176792.057612:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176792.057619:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000200:0.0:1786176792.057623:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176792.057625:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176792.057637:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176792.057639:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176792.057643:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.057648:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176792.057652:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176792.057654:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176792.057657:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176792.057662:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176792.057663:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176792.057694:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176792.057700:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7025)
00000100:00000001:0.0:1786176792.057701:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176792.057711:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.057720:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176792.057723:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.057725:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176792.057727:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.057729:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176792.057731:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176792.057734:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176792.057739:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176792.057741:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176792.057746:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176792.057756:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856298112:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176792.057761:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.057763:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176792.057766:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd500 x1872934856298112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.057773:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176792.057776:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176792.057779:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176792.057783:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176792.057785:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176792.057787:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176792.057789:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176792.057792:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:0.0:1786176792.057796:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176792.057798:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176792.057800:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd500.
00000100:00000001:0.0:1786176792.057803:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176792.057805:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.057807:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.057814:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176792.057816:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.057818:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176792.057820:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176792.057821:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176792.057825:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.057833:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176792.057834:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176792.057837:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176792.057839:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176792.057841:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176792.057844:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856298240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176792.057848:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176792.057850:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176792.057852:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176792.057854:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176792.057857:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176792.057859:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176792.057864:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176792.057866:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176792.057870:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176792.057874:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856298240, portal 10
00000100:00000001:0.0:1786176792.057876:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176792.057877:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178048 : -114548255373568 : ffff97d1a7edc700)
00000100:00000040:0.0:1786176792.057884:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.057891:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176792.057893:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176792.057896:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176792.057898:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856298240, offset 0
00000400:00000200:0.0:1786176792.057903:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176792.057907:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176792.057911:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176792.057914:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176792.057918:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176792.057920:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176792.057923:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176792.057926:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254501:254501:256:4294967295] 192.168.202.110@tcp LPNI seq info [254490:254490:8:4294967295]
00000400:00000200:0.0:1786176792.057930:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176792.057937:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176792.057942:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176792.057946:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176792.057950:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176792.057954:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176792.057958:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176792.057961:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786176792.057965:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176792.057967:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176792.057971:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176792.057974:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176792.057978:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176792.057980:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176792.057982:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176792.057986:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.057994:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176792.057998:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058004:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.058006:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176792.058009:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176792.058014:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176792.058018:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176792.058020:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301687818
00000100:00000001:0.0:1786176792.058023:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301691597
00000100:00000001:0.0:1786176792.058025:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301687818
00000100:00000001:0.0:1786176792.058027:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301691597
00000100:00000001:0.0:1786176792.058030:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176792.058032:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.058034:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176792.058039:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176792/real 1786176792]  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176792.058047:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176792.058048:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176792.058051:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176792.058054:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176792.058055:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176792.058057:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176792.058061:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176792.058070:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058077:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176792.058078:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.058080:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176792.058082:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176792.058086:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058093:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176792.058097:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058105:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176792.058108:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.058109:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176792.058111:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.058112:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176792.058114:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176792.058117:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176792.058120:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176792.058123:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176792.058127:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176792.058135:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856298240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176792.058139:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176792.058143:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058149:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176792.058151:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176792.058153:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176792.058155:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176792.058157:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176792.058160:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856298368:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176792.058164:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176792.058166:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176792.058168:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176792.058170:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176792.058173:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176792.058175:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176792.058179:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176792.058181:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176792.058184:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176792.058187:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856298368, portal 10
00000100:00000001:0.0:1786176792.058189:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176792.058190:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454180736 : -114548255370880 : ffff97d1a7edd180)
00000100:00000040:0.0:1786176792.058196:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058203:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176792.058205:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176792.058208:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176792.058210:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856298368, offset 0
00000400:00000200:0.0:1786176792.058214:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176792.058218:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176792.058222:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176792.058225:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176792.058228:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176792.058230:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176792.058233:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176792.058236:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254502:254502:256:4294967295] 192.168.202.110@tcp LPNI seq info [254491:254491:8:4294967295]
00000400:00000200:0.0:1786176792.058240:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176792.058247:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176792.058251:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176792.058255:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176792.058259:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176792.058262:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176792.058266:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176792.058269:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786176792.058273:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176792.058275:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176792.058278:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176792.058282:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176792.058285:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176792.058287:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176792.058289:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176792.058293:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058301:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176792.058304:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058311:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.058313:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176792.058316:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176792.058319:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176792.058323:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176792.058325:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301687943
00000100:00000001:0.0:1786176792.058327:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301691597
00000100:00000001:0.0:1786176792.058329:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301687943
00000100:00000001:0.0:1786176792.058332:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301691597
00000100:00000001:0.0:1786176792.058334:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176792.058337:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.058339:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176792.058343:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176792/real 1786176792]  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176792.058351:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176792.058353:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176792.058355:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176792.058357:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176792.058359:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176792.058361:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176792.058365:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176792.058373:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058380:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176792.058382:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.058384:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176792.058385:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176792.058389:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058396:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176792.058401:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058408:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176792.058410:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.058412:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176792.058413:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.058415:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176792.058418:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176792.058420:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176792.058424:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176792.058427:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176792.058431:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176792.058440:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856298368:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176792.058444:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176792.058447:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058454:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176792.058456:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176792.058458:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176792.058460:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176792.058462:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176792.058466:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856298496:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176792.058469:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176792.058471:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176792.058472:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176792.058474:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176792.058477:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176792.058479:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176792.058483:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176792.058485:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176792.058489:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176792.058492:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856298496, portal 4
00000100:00000001:0.0:1786176792.058494:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176792.058495:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187904 : -114548255363712 : ffff97d1a7eded80)
00000100:00000040:0.0:1786176792.058501:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058527:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176792.058531:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176792.058534:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176792.058537:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856298496, offset 0
00000400:00000200:0.0:1786176792.058541:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176792.058545:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176792.058549:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176792.058552:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176792.058554:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176792.058557:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176792.058559:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176792.058563:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254503:254503:256:4294967295] 192.168.202.110@tcp LPNI seq info [254492:254492:8:4294967295]
00000400:00000200:0.0:1786176792.058567:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176792.058573:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176792.058577:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176792.058581:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176792.058584:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176792.058587:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176792.058591:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176792.058594:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786176792.058598:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176792.058600:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176792.058603:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176792.058607:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176792.058610:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176792.058612:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176792.058614:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176792.058618:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058627:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176792.058630:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058643:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.058645:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176792.058648:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176792.058651:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176792.058655:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176792.058657:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301687819
00000100:00000001:0.0:1786176792.058659:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301691597
00000100:00000001:0.0:1786176792.058662:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301687819
00000100:00000001:0.0:1786176792.058664:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301691597
00000100:00000001:0.0:1786176792.058666:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176792.058669:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.058671:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176792.058675:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176792/real 1786176792]  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176792.058697:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176792.058698:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176792.058700:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176792.058703:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176792.058705:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176792.058708:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176792.058711:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176792.058720:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058727:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176792.058729:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.058731:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176792.058732:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176792.058736:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058743:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176792.058748:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058755:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176792.058758:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.058759:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176792.058761:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.058763:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176792.058765:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176792.058767:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176792.058771:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176792.058774:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176792.058777:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176792.058786:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856298496:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176792.058790:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176792.058794:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058801:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176792.058802:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176792.058805:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176792.058807:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176792.058809:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176792.058812:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856298624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176792.058815:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176792.058818:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176792.058820:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176792.058821:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176792.058825:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176792.058827:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176792.058830:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176792.058832:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176792.058835:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176792.058838:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856298624, portal 4
00000100:00000001:0.0:1786176792.058841:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176792.058842:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:0.0:1786176792.058848:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058855:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176792.058857:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176792.058859:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176792.058862:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856298624, offset 0
00000400:00000200:0.0:1786176792.058866:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176792.058870:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176792.058873:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176792.058876:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176792.058879:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176792.058881:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176792.058884:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176792.058888:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254504:254504:256:4294967295] 192.168.202.110@tcp LPNI seq info [254493:254493:8:4294967295]
00000400:00000200:0.0:1786176792.058892:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176792.058898:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176792.058903:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176792.058906:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176792.058910:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176792.058913:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786176792.058917:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176792.058920:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786176792.058924:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176792.058927:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176792.058930:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176792.058934:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176792.058937:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176792.058939:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176792.058941:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176792.058945:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058952:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176792.058956:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176817 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.058962:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.058964:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176792.058967:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176792.058970:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176792.058973:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176792.058975:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301687827
00000100:00000001:0.0:1786176792.058978:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301691597
00000100:00000001:0.0:1786176792.058980:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301687827
00000100:00000001:0.0:1786176792.058983:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301691597
00000100:00000001:0.0:1786176792.058985:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176792.058987:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.058989:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176792.058993:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176792/real 1786176792]  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176792.059001:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176792.059002:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176792.059004:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176792.059007:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176792.059009:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176792.059011:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176792.059014:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176792.059023:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.059030:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176792.059031:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.059033:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176792.059035:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176792.059038:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.059045:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176792.059049:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.059056:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176792.059058:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.059060:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176792.059062:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.059064:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176792.059065:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176792.059068:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176792.059072:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176792.059074:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176792.059078:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176792.059087:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856298624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176792.059090:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.059092:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176792.059097:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc700 x1872934856298240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.059103:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176792.059105:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176792.059107:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176792.059110:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176792.059112:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176792.059113:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176792.059115:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176792.059117:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:0.0:1786176792.059120:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176792.059122:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176792.059123:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc700.
00000100:00000001:0.0:1786176792.059126:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176792.059128:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.059130:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176792.059133:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd180 x1872934856298368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.059140:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176792.059141:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176792.059143:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176792.059145:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176792.059147:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176792.059148:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176792.059150:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176792.059152:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:0.0:1786176792.059154:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176792.059156:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176792.059157:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd180.
00000100:00000001:0.0:1786176792.059159:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176792.059160:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.059162:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176792.059166:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eded80 x1872934856298496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.059172:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176792.059173:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176792.059175:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176792.059177:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176792.059178:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176792.059180:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176792.059182:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176792.059183:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:0.0:1786176792.059185:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176792.059186:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176792.059188:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eded80.
00000100:00000001:0.0:1786176792.059190:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176792.059192:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.059193:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176792.059196:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856298624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176817 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176792.059203:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176792.059204:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176792.059205:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176792.059208:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176792.059209:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176792.059210:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176792.059212:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176792.059214:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:0.0:1786176792.059217:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176792.059218:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176792.059219:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:0.0:1786176792.059222:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176792.059223:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.059225:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176792.059229:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176792.059230:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176792.059232:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176792.059234:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176792.632712:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176792.632715:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176792.632718:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:2.0:1786176794.680692:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176794.680702:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176794.680705:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1786176795.704831:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176795.704837:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176795.704841:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176795.896616:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301691597, at 4301691648
00000100:00000010:0.0:1786176795.896654:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:2.0:1786176797.176668:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176797.176674:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176797.176678:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176797.176684:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176797.176689:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176797.176694:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176797.176698:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176797.176702:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176797.176706:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.176901:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176797.176908:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176797.176914:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176797.176927:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856298752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.176947:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176797.176951:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176797.176959:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176797.176965:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176797.176970:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176797.176975:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856298752:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176797.176985:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176797.176992:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176797.176998:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176797.177006:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176797.177018:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176797.177024:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176797.177035:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176797.177043:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176797.177053:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176797.177067:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856298752, portal 25
00000100:00000001:0.0:1786176797.177073:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176797.177078:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
00000100:00000040:0.0:1786176797.177091:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856298752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.177110:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176797.177119:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176797.177126:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176797.177135:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856298752, offset 0
00000400:00000200:0.0:1786176797.177150:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176797.177163:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176797.177172:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176797.177180:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176797.177187:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176797.177194:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176797.177198:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176797.177205:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254505:254505:256:4294967295] 192.168.202.110@tcp LPNI seq info [254494:254494:8:4294967295]
00000400:00000200:0.0:1786176797.177211:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176797.177226:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176797.177240:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176797.177248:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176797.177261:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176797.177266:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176797.177276:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176797.177281:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1786176797.177288:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176797.177293:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176797.177302:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176797.177310:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176797.177318:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176797.177323:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176797.177329:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176797.177337:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f800 x1872934856298752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.177352:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176797.177358:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856298752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.177375:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.177380:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176797.177391:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176797.177402:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176797.177408:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301696717
00000100:00000010:0.0:1786176797.177420:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176797.177433:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176797.177440:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301696717
00000100:00000010:0.0:1786176797.177449:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1851340c0.
00000100:00000001:0.0:1786176797.177454:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176797.177461:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176797.177465:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.177468:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176797.177477:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176797/real 1786176797]  req@ffff97d1ba69f800 x1872934856298752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176797.177500:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176797.177504:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176797.177507:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176797.177542:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176797.177547:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176797.177551:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176797.177558:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856298752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176797.177571:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856298752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.177583:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176797.177586:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.177589:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176797.177592:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176797.177600:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856298752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.177615:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176797.177623:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856298752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.177660:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176797.177667:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.177672:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176797.177675:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.177679:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176797.177682:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176797.177686:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176797.177693:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176797.177697:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176797.177704:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856298752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176797.177722:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856298752:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176797.177729:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.177734:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176797.177739:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856298752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.177752:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176797.177756:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176797.177759:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176797.177766:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176797.177769:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176797.177772:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786176797.177777:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176797.177780:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:0.0:1786176797.177787:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176797.177790:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176797.177793:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000001:0.0:1786176797.177797:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176797.177798:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.177801:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.177810:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176797.177812:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.177815:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176797.177817:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176797.177818:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176797.177823:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856298880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.177831:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176797.177833:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176797.177835:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176797.177838:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176797.177841:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176797.177844:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856298880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176797.177848:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176797.177850:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176797.177852:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176797.177854:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176797.177857:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176797.177858:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176797.177862:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176797.177863:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176797.177866:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176797.177869:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856298880, portal 10
00000100:00000001:0.0:1786176797.177870:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176797.177871:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
00000100:00000040:0.0:1786176797.177877:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856298880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.177882:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176797.177883:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176797.177885:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176797.177886:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856298880, offset 0
00000400:00000200:0.0:1786176797.177889:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176797.177892:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176797.177894:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176797.177896:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176797.177899:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176797.177901:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176797.177903:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176797.177905:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254506:254506:256:4294967295] 192.168.202.110@tcp LPNI seq info [254495:254495:8:4294967295]
00000400:00000200:0.0:1786176797.177908:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176797.177913:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176797.177915:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176797.177918:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176797.177921:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176797.177924:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176797.177929:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176797.177932:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1786176797.177935:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176797.177937:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176797.177940:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176797.177944:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176797.177947:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176797.177949:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176797.177952:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176797.177956:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856298880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.177965:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176797.177968:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856298880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.177973:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.177974:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176797.177977:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176797.177980:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786176797.177982:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176797.177984:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301692938
00000100:00000001:0.0:1786176797.177987:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301696717
00000100:00000001:0.0:1786176797.177988:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301692938
00000100:00000001:0.0:1786176797.177992:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301696717
00000100:00000001:0.0:1786176797.177993:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176797.177995:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.177996:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176797.177999:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176797/real 1786176797]  req@ffff97d1ba69d880 x1872934856298880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176797.178004:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176797.178005:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176797.178006:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176797.178007:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176797.178008:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176797.178010:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176797.178012:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856298880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176797.178017:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856298880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178022:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176797.178023:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178024:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176797.178025:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176797.178027:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856298880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178031:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176797.178033:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856298880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178037:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176797.178039:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178040:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176797.178040:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178042:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176797.178044:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176797.178046:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176797.178048:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176797.178050:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176797.178052:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856298880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176797.178057:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856298880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176797.178059:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176797.178062:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856299008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178066:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176797.178068:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176797.178069:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176797.178070:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176797.178071:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176797.178073:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856299008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176797.178075:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176797.178076:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176797.178077:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176797.178079:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176797.178081:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176797.178083:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176797.178085:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176797.178087:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176797.178089:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176797.178091:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856299008, portal 10
00000100:00000001:0.0:1786176797.178092:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176797.178093:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:0.0:1786176797.178097:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856299008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178103:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176797.178104:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176797.178106:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176797.178108:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856299008, offset 0
00000400:00000200:0.0:1786176797.178110:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176797.178113:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176797.178115:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176797.178117:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176797.178119:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176797.178120:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176797.178122:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176797.178124:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254507:254507:256:4294967295] 192.168.202.110@tcp LPNI seq info [254496:254496:8:4294967295]
00000400:00000200:0.0:1786176797.178127:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176797.178131:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176797.178134:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176797.178136:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176797.178139:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176797.178141:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176797.178144:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176797.178145:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1786176797.178148:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176797.178149:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176797.178151:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176797.178154:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176797.178156:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176797.178157:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176797.178159:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176797.178161:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856299008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178166:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176797.178168:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856299008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178178:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.178179:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176797.178181:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176797.178183:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786176797.178185:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176797.178186:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301693063
00000100:00000001:0.0:1786176797.178188:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301696717
00000100:00000001:0.0:1786176797.178190:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301693063
00000100:00000001:0.0:1786176797.178191:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301696717
00000100:00000001:0.0:1786176797.178192:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176797.178193:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.178194:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176797.178197:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176797/real 1786176797]  req@ffff97d1ba69ed80 x1872934856299008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176797.178201:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176797.178202:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176797.178203:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176797.178204:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176797.178205:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176797.178207:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176797.178209:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856299008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176797.178214:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856299008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178218:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176797.178219:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178220:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176797.178220:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176797.178222:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856299008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178227:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176797.178230:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856299008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178234:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176797.178236:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178237:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176797.178238:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178239:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176797.178240:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176797.178241:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176797.178243:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176797.178244:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176797.178246:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856299008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176797.178251:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856299008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176797.178253:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176797.178255:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856299136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178259:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176797.178260:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176797.178262:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176797.178263:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176797.178265:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176797.178266:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856299136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176797.178268:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176797.178269:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176797.178270:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176797.178272:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176797.178274:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176797.178275:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176797.178277:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176797.178279:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176797.178282:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176797.178283:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856299136, portal 4
00000100:00000001:0.0:1786176797.178284:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176797.178286:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00000040:0.0:1786176797.178290:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856299136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178295:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176797.178296:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176797.178297:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176797.178299:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856299136, offset 0
00000400:00000200:0.0:1786176797.178302:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176797.178304:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176797.178306:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176797.178308:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176797.178310:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176797.178311:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176797.178313:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176797.178315:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254508:254508:256:4294967295] 192.168.202.110@tcp LPNI seq info [254497:254497:8:4294967295]
00000400:00000200:0.0:1786176797.178318:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176797.178322:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176797.178324:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176797.178326:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176797.178328:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176797.178330:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176797.178332:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176797.178334:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1786176797.178337:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176797.178339:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176797.178340:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176797.178343:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176797.178344:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176797.178346:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176797.178347:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176797.178349:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856299136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178354:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176797.178356:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856299136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178361:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.178362:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176797.178363:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176797.178365:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786176797.178367:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176797.178369:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301692938
00000100:00000001:0.0:1786176797.178371:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301696717
00000100:00000001:0.0:1786176797.178374:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301692938
00000100:00000001:0.0:1786176797.178376:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301696717
00000100:00000001:0.0:1786176797.178379:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176797.178381:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.178383:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176797.178386:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176797/real 1786176797]  req@ffff97d1ba69e680 x1872934856299136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176797.178395:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176797.178396:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176797.178398:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176797.178401:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176797.178403:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176797.178406:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176797.178409:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856299136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176797.178419:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856299136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178426:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176797.178428:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178429:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176797.178431:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176797.178433:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856299136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178438:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176797.178441:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856299136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178445:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176797.178447:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178448:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176797.178449:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178450:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176797.178451:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176797.178452:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176797.178454:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176797.178455:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176797.178458:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856299136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176797.178462:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856299136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176797.178465:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176797.178467:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69dc00 x1872934856299264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178471:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176797.178472:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176797.178474:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176797.178476:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176797.178477:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176797.178478:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856299264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176797.178480:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176797.178481:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176797.178482:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176797.178484:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176797.178486:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176797.178487:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176797.178490:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176797.178491:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176797.178494:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176797.178496:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856299264, portal 4
00000100:00000001:0.0:1786176797.178497:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176797.178499:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764299776 : -114547945251840 : ffff97d1ba69dc00)
00000100:00000040:0.0:1786176797.178503:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69dc00 x1872934856299264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178507:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176797.178524:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176797.178527:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176797.178529:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856299264, offset 0
00000400:00000200:0.0:1786176797.178532:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176797.178534:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176797.178537:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176797.178539:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176797.178541:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176797.178543:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176797.178544:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176797.178547:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254509:254509:256:4294967295] 192.168.202.110@tcp LPNI seq info [254498:254498:8:4294967295]
00000400:00000200:0.0:1786176797.178550:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176797.178554:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176797.178557:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176797.178559:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176797.178562:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786176797.178564:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786176797.178567:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176797.178569:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1786176797.178571:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176797.178573:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786176797.178575:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176797.178577:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786176797.178580:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4000
00000400:00000010:0.0:1786176797.178581:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4000.
00000100:00000001:0.0:1786176797.178582:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176797.178585:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69dc00 x1872934856299264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178590:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176797.178592:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69dc00 x1872934856299264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178597:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.178598:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176797.178600:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176797.178602:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786176797.178604:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176797.178606:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301692947
00000100:00000001:0.0:1786176797.178607:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301696717
00000100:00000001:0.0:1786176797.178609:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301692947
00000100:00000001:0.0:1786176797.178611:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301696717
00000100:00000001:0.0:1786176797.178612:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176797.178614:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.178615:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786176797.178617:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176797/real 1786176797]  req@ffff97d1ba69dc00 x1872934856299264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786176797.178623:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786176797.178624:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786176797.178625:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c2c0.
00000400:00000200:0.0:1786176797.178638:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4b28
00000400:00000010:0.0:1786176797.178640:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4b28.
00000100:00000001:0.0:1786176797.178642:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786176797.178645:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69dc00 x1872934856299264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176797.178650:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69dc00 x1872934856299264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178655:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786176797.178656:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178657:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176797.178658:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786176797.178661:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69dc00 x1872934856299264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178665:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176797.178668:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69dc00 x1872934856299264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178672:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176797.178673:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178674:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786176797.178675:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178676:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176797.178677:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786176797.178679:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786176797.178681:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176797.178683:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786176797.178686:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69dc00 x1872934856299264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176797.178691:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856299264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786176797.178693:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178694:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176797.178696:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856298880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178700:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176797.178701:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176797.178702:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786176797.178704:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176797.178705:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176797.178706:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176797.178708:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176797.178709:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786176797.178710:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176797.178711:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176797.178712:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:0.0:1786176797.178714:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176797.178715:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178716:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176797.178719:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856299008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178723:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176797.178724:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176797.178725:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176797.178727:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176797.178727:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176797.178728:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786176797.178729:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176797.178731:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786176797.178732:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176797.178733:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176797.178734:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:0.0:1786176797.178736:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176797.178736:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178737:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176797.178739:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856299136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178743:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176797.178743:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176797.178744:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176797.178745:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176797.178746:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176797.178747:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176797.178748:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176797.178749:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786176797.178750:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176797.178751:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176797.178752:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:0.0:1786176797.178753:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176797.178754:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178755:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176797.178756:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69dc00 x1872934856299264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176797.178760:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786176797.178761:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786176797.178762:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786176797.178763:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786176797.178763:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786176797.178764:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786176797.178765:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786176797.178766:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786176797.178767:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786176797.178768:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786176797.178768:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69dc00.
00000100:00000001:0.0:1786176797.178770:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176797.178771:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178773:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176797.178775:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176797.178777:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176797.178778:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176797.178779:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176797.752779:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176797.752785:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176797.752789:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176798.200690:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176798.200699:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176798.200701:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176798.200707:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176798.200711:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176798.200714:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176798.200717:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176798.200719:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176798.200722:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176798.776732:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176798.776744:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176798.776748:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:0.0:1786176799.096919:0:2384:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00000001:2.0:1786176799.224636:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176799.224644:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176799.224646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176799.224651:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176799.224655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176799.224658:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176799.224661:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176799.224664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176799.224666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176800.248704:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176800.248714:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176800.248716:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176800.248722:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176800.248726:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176800.248730:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176800.248733:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176800.248735:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176800.248738:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176800.824742:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176800.824749:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176800.824755:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176801.016622:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301696717, at 4301696768
00000100:00000010:0.0:1786176801.016641:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:2.0:1786176801.272679:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176801.272689:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176801.272691:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176801.272698:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176801.272701:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176801.272705:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176801.272707:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176801.272710:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176801.272713:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176801.848771:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176801.848775:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176801.848779:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176802.296664:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176802.296674:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176802.296679:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176802.296688:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176802.296695:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176802.296701:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176802.296708:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176802.296711:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176802.296716:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176802.296929:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176802.296935:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176802.296939:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176802.296948:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c380 x1872934856299392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176802.296962:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176802.296965:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176802.296971:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176802.296976:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176802.296980:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176802.296987:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856299392:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176802.296996:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176802.297001:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176802.297006:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176802.297011:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176802.297017:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786176802.297023:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176802.297040:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176802.297050:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c2c0.
00000400:00000010:0.0:1786176802.297063:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4b28.
00000100:00000200:0.0:1786176802.297075:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856299392, portal 25
00000100:00000001:0.0:1786176802.297083:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176802.297088:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764293504 : -114547945258112 : ffff97d1ba69c380)
00000100:00000040:0.0:1786176802.297100:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c380 x1872934856299392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176802.297110:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176802.297113:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786176802.297118:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4000.
00000100:00000200:0.0:1786176802.297123:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856299392, offset 0
00000400:00000200:0.0:1786176802.297134:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176802.297146:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176802.297156:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786176802.297166:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786176802.297175:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786176802.297182:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786176802.297187:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786176802.297193:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254510:254510:256:4294967295] 192.168.202.110@tcp LPNI seq info [254499:254499:8:4294967295]
00000400:00000200:0.0:1786176802.297200:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176802.297218:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786176802.297228:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786176802.297235:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786176802.297249:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786176802.297257:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786176802.297277:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176802.297287:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176802.297291:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301701837
00000100:00000010:0.0:1786176802.297301:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786176802.297310:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176802.297318:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301701837
00000100:00000010:0.0:1786176802.297326:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134d80.
00000100:00000001:0.0:1786176802.297331:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176802.297337:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176802.297342:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176802.297347:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176802.297353:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176802.297364:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176802.297367:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1786176802.297373:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176802.297377:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176802.297381:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176802.297388:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69dc00 x1872934856299520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176802.297406:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176802.297409:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176802.297413:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176802.297417:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176802.297423:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176802.297429:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856299520:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176802.297436:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176802.297438:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176802.297440:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176802.297444:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786176802.297449:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786176802.297455:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786176802.297462:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786176802.297468:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381cd68.
00000400:00000010:0.0:1786176802.297474:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4880.
00000100:00000200:0.0:1786176802.297479:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856299520, portal 10
00000100:00000001:0.0:1786176802.297483:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176802.297486:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764299776 : -114547945251840 : ffff97d1ba69dc00)
00000100:00000040:0.0:1786176802.297495:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69dc00 x1872934856299520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176802.297507:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176802.297542:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000800:00000200:1.1:1786176802.297689:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176802.297855:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176802.298033:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176802.298203:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176803.320677:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176803.320690:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176803.320694:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176803.320703:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176803.320708:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176803.320713:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176803.320716:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176803.320720:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176803.320724:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176803.896721:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176803.896729:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176803.896734:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176804.344641:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176804.344650:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176804.344653:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176804.344659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176804.344664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176804.344669:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176804.344672:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176804.344676:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176804.344679:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176804.920837:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176804.920840:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176804.920843:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176805.368646:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176805.368654:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176805.368657:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176805.368663:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176805.368668:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176805.368672:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176805.368675:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176805.368679:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176805.368682:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176806.136611:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301701837, at 4301701888
00000100:00000010:0.0:1786176806.136632:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:2.0:1786176806.392637:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176806.392646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176806.392648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176806.392653:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176806.392657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176806.392660:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176806.392663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176806.392666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176806.392668:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176806.968747:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176806.968753:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176806.968757:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176807.416688:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176807.416693:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176807.416696:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176807.416702:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176807.416706:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176807.416709:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176807.416712:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176807.416717:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176807.416722:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176807.417781:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176807.417785:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786176807.418193:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176807.418196:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786176807.418594:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176807.418597:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786176807.418818:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176807.418821:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786176807.419037:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176807.419039:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:0.0:1786176807.992760:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176807.992764:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176807.992767:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176808.440711:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176808.440720:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176808.440722:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176808.440728:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176808.440732:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176808.440737:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176808.440740:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176808.440743:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176808.440746:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176809.464587:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176809.464593:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176809.464596:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176809.464601:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176809.464605:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176809.464608:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176809.464611:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176809.464613:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176809.464616:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176810.040719:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176810.040723:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176810.040726:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176810.488623:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176810.488631:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176810.488633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176810.488637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176810.488640:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176810.488643:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176810.488646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176810.488648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176810.488650:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176811.064677:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176811.064680:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176811.064683:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176811.256632:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301706958, at 4301707008
00000100:00000010:0.0:1786176811.256653:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37600.
00000100:00000001:2.0:1786176811.512574:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176811.512581:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176811.512583:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176811.512587:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176811.512590:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176811.512593:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176811.512595:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176811.512598:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176811.512600:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176812.536617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176812.536623:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176812.536627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176812.536633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176812.536637:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176812.536641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176812.536644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176812.536648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176812.536653:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176812.537297:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176812.537304:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786176812.537948:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176812.537951:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786176812.538176:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176812.538178:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786176812.538354:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176812.538356:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786176812.538597:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176812.538600:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:2.0:1786176813.112671:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176813.112674:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176813.112677:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176813.560620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176813.560629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176813.560632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176813.560648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176813.560652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176813.560656:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176813.560659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176813.560662:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176813.560665:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176814.136703:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176814.136706:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176814.136709:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176814.584608:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176814.584615:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176814.584616:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176814.584621:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176814.584624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176814.584627:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176814.584629:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176814.584631:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176814.584633:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176815.608600:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176815.608608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176815.608610:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176815.608614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176815.608617:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176815.608620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176815.608623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176815.608625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176815.608627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176816.184692:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176816.184695:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176816.184698:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176816.376638:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301712077, at 4301712128
00000100:00000010:0.0:1786176816.376657:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37c00.
00000100:00000001:2.0:1786176816.632586:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176816.632592:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176816.632594:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176816.632598:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176816.632601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176816.632604:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176816.632606:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176816.632608:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176816.632610:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176817.208720:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176817.208724:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176817.208727:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176817.656617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176817.656624:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176817.656626:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176817.656630:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176817.656634:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176817.656637:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176817.656640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176817.656643:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176817.656647:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176817.657154:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176817.657160:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786176817.657800:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176817.657802:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786176817.658042:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176817.658043:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786176817.658256:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176817.658258:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786176817.658489:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176817.658491:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:2.0:1786176818.680587:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176818.680593:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176818.680594:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176818.680599:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176818.680601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176818.680604:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176818.680607:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176818.680609:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176818.680611:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176819.256784:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176819.256789:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176819.256792:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176819.704658:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176819.704666:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176819.704669:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176819.704675:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176819.704679:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176819.704683:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176819.704687:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176819.704691:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176819.704695:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176820.280678:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176820.280682:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176820.280684:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176820.728659:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176820.728670:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176820.728672:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176820.728680:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176820.728686:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176820.728691:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176820.728695:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176820.728698:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176820.728702:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176821.496620:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301717197, at 4301717248
00000100:00000010:0.0:1786176821.496634:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37720.
00000100:00000001:2.0:1786176821.752684:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176821.752695:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176821.752697:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176821.752705:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176821.752709:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176821.752712:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176821.752715:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176821.752718:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176821.752721:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176822.328698:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176822.328705:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176822.328708:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176822.776670:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176822.776682:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176822.776691:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176822.776701:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176822.776709:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176822.776720:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176822.776726:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176822.776734:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176822.776749:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176822.777411:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176822.777419:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786176822.777974:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176822.777977:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786176822.778296:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176822.778298:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786176822.778658:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176822.778661:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786176822.778966:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176822.778969:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:0.0:1786176823.352687:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176823.352696:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176823.352701:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176823.800653:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176823.800661:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176823.800663:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176823.800669:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176823.800672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176823.800675:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176823.800678:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176823.800680:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176823.800683:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176824.824590:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176824.824597:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176824.824598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176824.824603:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176824.824606:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176824.824609:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176824.824611:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176824.824613:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176824.824615:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176825.400809:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176825.400813:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176825.400817:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176825.848707:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176825.848718:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176825.848722:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176825.848730:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176825.848735:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176825.848740:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176825.848744:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176825.848748:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176825.848752:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176826.424718:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176826.424724:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176826.424728:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176826.616650:0:104806:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301722318, at 4301722368
00000100:00000010:0.0:1786176826.616679:0:104806:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37960.
00000100:00000001:2.0:1786176826.872643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176826.872652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176826.872654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176826.872660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176826.872663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176826.872666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176826.872669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176826.872671:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176826.872674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176827.896649:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786176827.896656:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:0.0:1786176827.896658:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786176827.896663:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1786176827.896667:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786176827.896668:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176827.896672:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.896673:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:2.0:1786176827.896681:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:0.0:1786176827.896682:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7051
00000100:00000001:2.0:1786176827.896692:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000400:00000200:0.0:1786176827.896697:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176827.896700:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1786176827.896707:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1786176827.896709:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1786176827.896711:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1786176827.896715:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176827.896716:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.896725:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1786176827.896726:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176827.896728:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.896735:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176827.896738:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176827.896741:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786176827.896743:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176827.896746:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786176827.896748:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786176827.896750:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786176827.896752:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.896760:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176827.896765:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.896766:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176827.896770:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786176827.896772:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176827.896774:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176827.896776:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176827.896779:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176827.896781:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.896782:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786176827.896785:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176827.896793:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a300.
00000020:00000040:0.0:1786176827.896796:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786176827.896799:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176827.896801:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.896803:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176827.896805:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176827.896809:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176827.896815:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000100:00000001:0.0:1786176827.896820:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176827.896827:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a300 x1872934856302592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176827.896835:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176827.896838:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176827.896841:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b0d3569] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a300 x1872934856302592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176827.896858:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176827.896861:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176827.896863:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786176827.896864:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176827.896865:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176827.896867:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176827.896869:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7051
00000400:00000200:0.0:1786176827.896871:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176827.896874:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176827.896876:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176827.896878:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.896880:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176827.896882:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.896884:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176827.896886:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176827.896887:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786176827.896889:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176827.896891:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786176827.896895:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786176827.896898:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786176827.896902:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.896904:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176827.896907:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.896910:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176827.896913:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786176827.896918:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176827.896922:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176827.896926:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176827.896933:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176827.896936:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.896939:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176827.896944:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786176827.896949:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.896953:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176827.896958:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41bb80.
00000020:00000040:0.0:1786176827.896962:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786176827.896967:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176827.896969:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.896971:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176827.896973:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176827.896974:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176827.896978:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000100:00000001:0.0:1786176827.896982:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176827.896989:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41bb80 x1872934856302720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176827.896998:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176827.897001:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176827.897006:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9983a05] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41bb80 x1872934856302720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176827.897021:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176827.897025:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176827.897032:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786176827.897035:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176827.897037:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176827.897041:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176827.897047:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7051
00000400:00000200:0.0:1786176827.897053:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176827.897058:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176827.897061:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176827.897066:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897069:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176827.897071:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897073:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176827.897074:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176827.897076:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786176827.897079:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176827.897082:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786176827.897083:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786176827.897086:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786176827.897091:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897095:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176827.897099:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897102:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176827.897106:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786176827.897113:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897118:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176827.897123:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176827.897125:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176827.897129:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.897132:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176827.897134:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786176827.897136:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.897140:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176827.897145:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419c00.
00000020:00000040:0.0:1786176827.897148:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786176827.897153:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176827.897156:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.897158:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176827.897159:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176827.897160:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176827.897167:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7800.
00000100:00000001:0.0:1786176827.897170:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176827.897176:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419c00 x1872934856302848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176827.897187:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176827.897190:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176827.897194:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786176827.897195:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cf890c1c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419c00 x1872934856302848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1786176827.897202:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:0.0:1786176827.897204:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176827.897209:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176827.897212:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786176827.897214:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176827.897217:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176827.897221:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176827.897224:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7051
00000400:00000200:0.0:1786176827.897229:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176827.897232:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176827.897237:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176827.897243:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897246:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176827.897248:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897250:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176827.897255:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176827.897256:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786176827.897259:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176827.897262:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786176827.897264:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786176827.897265:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786176827.897270:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897271:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176827.897275:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897277:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176827.897282:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786176827.897286:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897291:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176827.897294:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176827.897296:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176827.897298:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.897300:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176827.897305:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786176827.897308:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.897313:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176827.897317:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a680.
00000020:00000040:0.0:1786176827.897321:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786176827.897324:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176827.897327:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.897355:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176827.897358:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176827.897360:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176827.897364:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000100:00000001:0.0:1786176827.897370:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176827.897377:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a680 x1872934856302976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176827.897387:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176827.897390:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176827.897396:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000911ee1e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a680 x1872934856302976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176827.897417:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176827.897422:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176827.897425:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786176827.897428:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176827.897431:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176827.897436:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176827.897441:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7051
00000400:00000200:0.0:1786176827.897447:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176827.897453:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176827.897462:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176827.897467:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897471:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176827.897472:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897474:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176827.897475:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176827.897476:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786176827.897479:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176827.897482:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786176827.897486:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:0.0:1786176827.897491:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786176827.897496:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897498:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176827.897503:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897507:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176827.897552:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786176827.897558:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176827.897564:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176827.897567:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176827.897569:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176827.897571:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.897572:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176827.897574:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786176827.897578:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.897584:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176827.897590:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419500.
00000020:00000040:0.0:1786176827.897594:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786176827.897597:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176827.897599:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176827.897603:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176827.897605:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176827.897607:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176827.897611:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5800.
00000100:00000001:0.0:1786176827.897615:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176827.897623:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419500 x1872934856303104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176827.897640:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176827.897644:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176827.897651:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b6718a9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419500 x1872934856303104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176827.897662:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176827.897667:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7061)
00000800:00000200:3.1:1786176827.897853:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176827.897855:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176827.898049:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176827.898051:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176827.898283:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176827.898286:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786176827.898553:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176827.898556:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:2.0:1786176828.472698:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176828.472704:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176828.472708:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176828.920566:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176828.920577:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176828.920579:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176828.920586:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176828.920589:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176828.920592:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176828.920595:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176828.920598:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176828.920600:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176829.496721:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176829.496729:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176829.496735:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176829.944643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176829.944652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176829.944654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176829.944660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176829.944663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176829.944666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176829.944669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176829.944671:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176829.944674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176830.968657:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176830.968666:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176830.968669:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176830.968675:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176830.968679:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176830.968683:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176830.968686:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176830.968688:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176830.968691:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176831.544719:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176831.544724:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176831.544727:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176831.736592:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301727437, at 4301727488
00000100:00000010:0.0:1786176831.736618:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37240.
00000100:00000001:2.0:1786176831.992704:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176831.992713:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176831.992716:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176831.992731:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176831.992735:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176831.992739:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176831.992742:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176831.992745:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176831.992748:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176832.568710:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176832.568716:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176832.568720:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176833.016688:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176833.016699:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176833.016702:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176833.016711:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176833.016716:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176833.016722:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176833.016726:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176833.016730:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176833.016734:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176833.016766:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176833.016773:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176833.016777:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176833.016782:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176833.016787:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176833.016797:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7056
00000400:00000200:2.0:1786176833.016808:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176833.016817:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176833.016821:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176833.016825:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.016830:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176833.016833:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.016837:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176833.016842:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176833.016845:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176833.016849:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176833.016853:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176833.016856:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176833.016860:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176833.016864:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.016866:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176833.016868:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.016871:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176833.016874:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176833.016878:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176833.016881:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176833.016885:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176833.016891:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176833.016894:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.016897:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176833.016900:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176833.016913:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786176833.016918:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176833.016924:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176833.016927:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.016931:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176833.016935:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176833.016941:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176833.016954:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786176833.016964:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176833.016976:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856303232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176833.016989:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176833.016996:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176833.017002:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856303232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176833.017033:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176833.017037:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176833.017041:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176833.017043:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176833.017045:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176833.017048:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176833.017051:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7056
00000400:00000200:2.0:1786176833.017056:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176833.017059:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176833.017062:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176833.017067:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.017069:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176833.017071:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.017073:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176833.017076:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176833.017078:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176833.017080:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176833.017083:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176833.017086:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176833.017089:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176833.017093:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.017104:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176833.017107:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.017109:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176833.017114:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176833.017122:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176833.017132:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176833.017138:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176833.017147:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176833.017160:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.017171:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176833.017183:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176833.017192:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.017203:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176833.017216:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786176833.017225:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176833.017235:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176833.017238:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.017242:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176833.017248:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176833.017254:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176833.017269:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000001:2.0:1786176833.017278:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176833.017306:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856303360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176833.017338:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176833.017349:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176833.017376:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856303360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176833.017411:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176833.017425:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176833.017445:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176833.017457:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176833.017461:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176833.017465:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176833.017471:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7056
00000400:00000200:2.0:1786176833.017489:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176833.017502:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176833.017509:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176833.017553:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.017559:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176833.017566:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.017569:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176833.017576:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000200:3.1:1786176833.017577:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1786176833.017578:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176833.017583:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176833.017588:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000800:00000200:3.1:1786176833.017589:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:2.0:1786176833.017591:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176833.017595:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176833.017607:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.017613:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176833.017620:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.017624:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176833.017629:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176833.017633:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176833.017649:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176833.017653:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176833.017659:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176833.017662:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.017671:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176833.017676:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176833.017679:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.017680:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176833.017683:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786176833.017687:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176833.017693:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176833.017696:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.017705:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176833.017708:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176833.017711:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176833.017720:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786176833.017730:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176833.017745:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856303488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176833.017770:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176833.017778:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176833.017791:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856303488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176833.017831:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176833.017843:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176833.017861:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176833.017867:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176833.017874:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176833.017886:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176833.017895:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7056
00000400:00000200:2.0:1786176833.017912:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176833.017928:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176833.017938:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176833.017956:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.017970:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176833.017983:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.017995:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176833.018008:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176833.018017:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176833.018023:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176833.018031:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176833.018041:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176833.018052:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176833.018068:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.018080:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176833.018090:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.018104:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176833.018113:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176833.018131:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176833.018147:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176833.018157:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176833.018168:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176833.018175:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.018185:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176833.018189:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176833.018203:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.018211:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176833.018224:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176833.018236:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176833.018248:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176833.018254:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.018262:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176833.018270:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176833.018276:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176833.018302:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786176833.018315:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176833.018327:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856303616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176833.018353:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176833.018362:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176833.018380:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856303616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176833.018437:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176833.018452:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176833.018467:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176833.018475:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176833.018481:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176833.018488:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176833.018497:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7056
00000800:00000200:3.1:1786176833.018503:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176833.018507:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000400:00000200:2.0:1786176833.018545:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176833.018562:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176833.018569:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176833.018578:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.018587:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176833.018592:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.018600:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176833.018606:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176833.018611:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176833.018621:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176833.018630:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176833.018639:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176833.018649:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176833.018665:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.018673:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176833.018685:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176833.018694:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176833.018703:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786176833.018723:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176833.018736:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176833.018747:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176833.018758:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176833.018771:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.018780:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176833.018789:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176833.018800:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.018809:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176833.018821:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786176833.018829:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176833.018838:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176833.018846:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176833.018855:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176833.018862:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176833.018870:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176833.018883:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786176833.018895:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176833.018911:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856303744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176833.018982:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176833.018990:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176833.019007:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856303744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176833.019056:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176833.019072:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7066)
00000800:00000200:3.1:1786176833.019167:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176833.019168:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176833.019430:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176833.019432:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786176833.019658:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176833.019660:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1786176834.040603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176834.040611:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176834.040613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176834.040618:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176834.040622:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176834.040625:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176834.040627:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176834.040629:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176834.040632:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176834.616744:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176834.616750:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176834.616754:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176835.064630:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176835.064639:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176835.064642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176835.064648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176835.064652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176835.064657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176835.064661:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176835.064664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176835.064666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176835.640756:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176835.640764:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176835.640770:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176836.088665:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176836.088677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176836.088680:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176836.088688:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176836.088693:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176836.088699:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176836.088703:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176836.088707:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176836.088711:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176836.856624:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301732558, at 4301732608
00000100:00000010:0.0:1786176836.856642:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37a20.
00000800:00000200:0.0:1786176836.984811:0:2384:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00000001:2.0:1786176837.112683:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176837.112692:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176837.112695:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176837.112700:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176837.112705:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176837.112709:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176837.112712:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176837.112716:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176837.112719:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176837.688644:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176837.688647:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176837.688650:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176838.136613:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176838.136621:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176838.136623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176838.136629:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176838.136633:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176838.136637:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176838.136641:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176838.136644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176838.136647:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786176838.137181:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176838.137186:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:3.0:1786176838.137769:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176838.137771:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176838.137773:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856304000, offset 0
00000400:00000200:3.0:1786176838.137776:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176838.137779:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176838.137782:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176838.137784:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176838.137787:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176838.137789:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176838.137792:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176838.137795:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254546:254546:256:4294967295] 192.168.202.110@tcp LPNI seq info [254535:254535:8:4294967295]
00000400:00000200:3.0:1786176838.137799:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176838.137806:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176838.137812:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176838.137816:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176838.137820:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176838.137823:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176838.137829:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176838.137832:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786176838.137836:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176838.137838:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176838.137841:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176838.137843:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176838.137846:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176838.137847:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176838.137849:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176838.137852:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa5180 x1872934856304000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.137858:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176838.137860:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5180 x1872934856304000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176863 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.137864:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176838.137865:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176838.137868:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176838.137871:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786176838.137873:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176838.137875:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301733898
00000100:00000001:3.0:1786176838.137877:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301737677
00000100:00000001:3.0:1786176838.137879:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301733898
00000100:00000001:3.0:1786176838.137880:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301737677
00000100:00000001:3.0:1786176838.137881:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176838.137883:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176838.137884:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176838.137886:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176838/real 1786176838]  req@ffff97d1b9aa5180 x1872934856304000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176838.137895:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176838.137896:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176838.137898:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176838.137901:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176838.137902:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176838.137905:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176838.137908:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5180 x1872934856304000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176838.137917:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5180 x1872934856304000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.137925:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176838.137927:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.137929:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176838.137930:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176838.137934:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5180 x1872934856304000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.137941:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176838.137944:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5180 x1872934856304000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.137949:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176838.137951:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.137952:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176838.137953:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.137955:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176838.137956:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176838.137957:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176838.137959:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176838.137961:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176838.137963:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5180 x1872934856304000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176838.137968:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786176838.137971:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176838.137973:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4e00 x1872934856304128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.137977:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176838.137978:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176838.137979:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176838.137981:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176838.137982:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176838.137984:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786176838.137986:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176838.137987:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176838.137988:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176838.137990:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176838.137992:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786176838.137994:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176838.137996:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176838.137998:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176838.138000:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176838.138003:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856304128, portal 10
00000100:00000001:3.0:1786176838.138004:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176838.138005:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751746048 : -114547957805568 : ffff97d1b9aa4e00)
00000100:00000040:3.0:1786176838.138009:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4e00 x1872934856304128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138013:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176838.138015:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176838.138016:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176838.138017:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856304128, offset 0
00000400:00000200:3.0:1786176838.138020:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176838.138022:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176838.138025:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176838.138027:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176838.138028:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176838.138030:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176838.138032:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176838.138034:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254547:254547:256:4294967295] 192.168.202.110@tcp LPNI seq info [254536:254536:8:4294967295]
00000400:00000200:3.0:1786176838.138037:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176838.138041:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176838.138043:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176838.138045:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176838.138048:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176838.138050:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176838.138053:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176838.138055:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786176838.138057:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176838.138058:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176838.138060:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176838.138063:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176838.138065:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176838.138066:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176838.138068:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176838.138070:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa4e00 x1872934856304128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138076:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176838.138078:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4e00 x1872934856304128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176863 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138082:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176838.138083:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176838.138085:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176838.138087:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786176838.138089:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176838.138090:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301734023
00000100:00000001:3.0:1786176838.138092:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301737677
00000100:00000001:3.0:1786176838.138093:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301734023
00000100:00000001:3.0:1786176838.138095:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301737677
00000100:00000001:3.0:1786176838.138096:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176838.138097:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176838.138098:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176838.138100:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176838/real 1786176838]  req@ffff97d1b9aa4e00 x1872934856304128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176838.138105:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176838.138106:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176838.138107:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176838.138109:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176838.138110:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176838.138111:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176838.138113:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4e00 x1872934856304128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176838.138119:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4e00 x1872934856304128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138124:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176838.138124:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138126:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176838.138127:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176838.138129:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4e00 x1872934856304128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138133:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176838.138135:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4e00 x1872934856304128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138139:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176838.138141:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138142:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176838.138142:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138143:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176838.138145:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176838.138146:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176838.138148:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176838.138149:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176838.138151:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4e00 x1872934856304128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176838.138156:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786176838.138158:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176838.138160:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa7b80 x1872934856304256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138164:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176838.138165:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176838.138167:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176838.138168:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176838.138170:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176838.138171:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786176838.138174:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176838.138174:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176838.138175:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176838.138176:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176838.138178:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786176838.138179:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176838.138182:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176838.138183:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176838.138185:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176838.138187:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856304256, portal 4
00000100:00000001:3.0:1786176838.138189:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176838.138190:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751757696 : -114547957793920 : ffff97d1b9aa7b80)
00000100:00000040:3.0:1786176838.138193:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa7b80 x1872934856304256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138197:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176838.138199:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176838.138200:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176838.138202:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856304256, offset 0
00000400:00000200:3.0:1786176838.138204:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176838.138206:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176838.138208:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176838.138209:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176838.138211:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176838.138213:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176838.138214:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176838.138216:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254548:254548:256:4294967295] 192.168.202.110@tcp LPNI seq info [254537:254537:8:4294967295]
00000400:00000200:3.0:1786176838.138219:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176838.138223:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176838.138226:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176838.138228:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176838.138245:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176838.138247:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176838.138250:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176838.138252:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786176838.138255:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176838.138256:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176838.138259:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176838.138261:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176838.138263:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176838.138265:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176838.138267:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176838.138269:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa7b80 x1872934856304256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138274:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176838.138277:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7b80 x1872934856304256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176863 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138284:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176838.138286:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176838.138289:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176838.138292:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786176838.138296:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176838.138298:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301733898
00000100:00000001:3.0:1786176838.138301:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301737677
00000100:00000001:3.0:1786176838.138303:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301733898
00000100:00000001:3.0:1786176838.138305:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301737677
00000100:00000001:3.0:1786176838.138307:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176838.138310:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176838.138312:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176838.138316:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176838/real 1786176838]  req@ffff97d1b9aa7b80 x1872934856304256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176838.138326:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176838.138327:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176838.138329:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176838.138331:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176838.138332:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176838.138334:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176838.138336:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa7b80 x1872934856304256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176838.138342:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa7b80 x1872934856304256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138346:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176838.138347:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138348:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176838.138349:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176838.138351:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa7b80 x1872934856304256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138358:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176838.138362:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa7b80 x1872934856304256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138369:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176838.138372:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138373:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176838.138375:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138377:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176838.138378:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176838.138381:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176838.138385:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176838.138388:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176838.138393:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa7b80 x1872934856304256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176838.138402:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786176838.138405:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176838.138407:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa6d80 x1872934856304384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138411:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176838.138412:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176838.138414:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176838.138415:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176838.138417:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176838.138418:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786176838.138420:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176838.138421:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176838.138422:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176838.138423:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176838.138426:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786176838.138427:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176838.138430:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176838.138432:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176838.138434:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176838.138436:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856304384, portal 4
00000100:00000001:3.0:1786176838.138437:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176838.138438:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751754112 : -114547957797504 : ffff97d1b9aa6d80)
00000100:00000040:3.0:1786176838.138442:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa6d80 x1872934856304384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138446:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176838.138448:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176838.138449:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176838.138451:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856304384, offset 0
00000400:00000200:3.0:1786176838.138454:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176838.138456:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176838.138459:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176838.138461:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176838.138463:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176838.138465:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176838.138466:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176838.138468:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254549:254549:256:4294967295] 192.168.202.110@tcp LPNI seq info [254538:254538:8:4294967295]
00000400:00000200:3.0:1786176838.138471:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176838.138475:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176838.138478:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176838.138480:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176838.138483:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176838.138485:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786176838.138488:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176838.138489:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786176838.138492:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176838.138494:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176838.138496:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176838.138498:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176838.138500:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176838.138501:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176838.138502:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176838.138505:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa6d80 x1872934856304384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176863 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138525:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176838.138528:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa6d80 x1872934856304384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176863 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138532:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176838.138533:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176838.138535:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176838.138537:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786176838.138540:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176838.138541:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301733907
00000100:00000001:3.0:1786176838.138543:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301737677
00000100:00000001:3.0:1786176838.138545:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301733907
00000100:00000001:3.0:1786176838.138546:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301737677
00000100:00000001:3.0:1786176838.138547:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176838.138548:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176838.138549:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176838.138552:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176838/real 1786176838]  req@ffff97d1b9aa6d80 x1872934856304384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176838.138556:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176838.138557:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176838.138558:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176838.138560:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176838.138561:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176838.138562:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176838.138564:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa6d80 x1872934856304384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176838.138569:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa6d80 x1872934856304384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138573:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176838.138574:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138575:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176838.138576:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176838.138578:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa6d80 x1872934856304384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138582:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176838.138584:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa6d80 x1872934856304384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138589:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176838.138591:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138592:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176838.138593:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138594:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176838.138595:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176838.138597:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176838.138599:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176838.138600:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176838.138602:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa6d80 x1872934856304384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176838.138607:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786176838.138609:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138610:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176838.138612:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5180 x1872934856304000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138617:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176838.138618:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176838.138619:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786176838.138621:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176838.138622:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176838.138623:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786176838.138624:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176838.138626:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03400.
02000000:00000001:3.0:1786176838.138627:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176838.138628:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176838.138629:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5180.
00000100:00000001:3.0:1786176838.138631:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176838.138632:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138633:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176838.138635:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4e00 x1872934856304128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138644:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176838.138645:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176838.138646:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786176838.138647:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176838.138648:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176838.138648:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786176838.138649:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176838.138650:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03000.
02000000:00000001:3.0:1786176838.138651:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176838.138652:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176838.138653:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4e00.
00000100:00000001:3.0:1786176838.138654:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176838.138655:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138656:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176838.138658:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa7b80 x1872934856304256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138661:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176838.138662:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176838.138663:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786176838.138664:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176838.138664:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176838.138665:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786176838.138666:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176838.138667:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b01800.
02000000:00000001:3.0:1786176838.138669:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176838.138670:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176838.138671:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa7b80.
00000100:00000001:3.0:1786176838.138672:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176838.138673:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138674:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176838.138675:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa6d80 x1872934856304384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176863 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176838.138679:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176838.138680:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176838.138681:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786176838.138683:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176838.138684:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176838.138685:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786176838.138686:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176838.138687:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00800.
02000000:00000001:3.0:1786176838.138688:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176838.138689:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176838.138690:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa6d80.
00000100:00000001:3.0:1786176838.138691:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176838.138692:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138693:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176838.138695:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176838.138696:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176838.138697:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786176838.138699:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176838.712697:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176838.712703:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176838.712706:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176839.160627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176839.160634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176839.160636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176839.160641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176839.160644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176839.160648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176839.160650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176839.160652:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176839.160655:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176839.736733:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176839.736738:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176839.736755:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176840.184625:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176840.184645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176840.184648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176840.184653:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176840.184657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176840.184662:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176840.184665:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176840.184668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176840.184672:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176840.760711:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176840.760714:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176840.760717:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176841.208642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176841.208652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176841.208654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176841.208660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176841.208664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176841.208667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176841.208670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176841.208672:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176841.208674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176841.784773:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176841.784777:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176841.784780:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176841.976647:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301737677, at 4301737728
00000100:00000010:0.0:1786176841.976667:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37f00.
00000100:00000001:2.0:1786176842.232653:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176842.232660:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176842.232663:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176842.232668:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176842.232672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176842.232676:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176842.232679:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176842.232682:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176842.232685:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176842.808885:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176842.808891:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176842.808895:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176843.256711:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176843.256721:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176843.256725:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176843.256736:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176843.256744:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176843.256751:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176843.256758:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176843.256764:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176843.256771:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.257129:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786176843.257137:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176843.257146:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176843.257162:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4380 x1872934856304512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.257197:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176843.257202:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176843.257215:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176843.257222:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176843.257229:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176843.257241:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856304512:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786176843.257253:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176843.257262:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176843.257267:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176843.257275:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176843.257284:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786176843.257290:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176843.257303:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176843.257311:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176843.257324:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176843.257338:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856304512, portal 25
00000100:00000001:3.0:1786176843.257345:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176843.257351:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751743360 : -114547957808256 : ffff97d1b9aa4380)
00000100:00000040:3.0:1786176843.257364:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4380 x1872934856304512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.257378:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176843.257385:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176843.257390:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176843.257396:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856304512, offset 0
00000400:00000200:3.0:1786176843.257409:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176843.257424:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176843.257435:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176843.257443:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176843.257452:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176843.257459:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176843.257464:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176843.257473:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254550:254550:256:4294967295] 192.168.202.110@tcp LPNI seq info [254539:254539:8:4294967295]
00000400:00000200:3.0:1786176843.257483:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176843.257495:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176843.257505:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176843.257545:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176843.257557:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176843.257562:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786176843.257573:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176843.257579:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1786176843.257586:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176843.257590:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176843.257598:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176843.257606:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176843.257615:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176843.257619:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176843.257626:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176843.257634:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa4380 x1872934856304512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.257653:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176843.257660:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4380 x1872934856304512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.257674:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.257676:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176843.257684:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176843.257693:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176843.257698:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301742798
00000100:00000010:3.0:1786176843.257705:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad374e0.
00000100:00000001:3.0:1786176843.257716:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786176843.257724:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301742798
00000100:00000010:3.0:1786176843.257730:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37420.
00000100:00000001:3.0:1786176843.257736:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786176843.257742:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176843.257747:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.257751:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176843.257758:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176843/real 1786176843]  req@ffff97d1b9aa4380 x1872934856304512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176843.257774:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176843.257777:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176843.257781:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176843.257786:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176843.257788:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176843.257792:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176843.257798:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4380 x1872934856304512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176843.257812:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4380 x1872934856304512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.257837:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176843.257840:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.257844:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176843.257848:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176843.257856:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4380 x1872934856304512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.257874:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176843.257883:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4380 x1872934856304512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.257895:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176843.257899:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.257902:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176843.257906:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.257912:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176843.257915:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176843.257918:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176843.257924:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176843.257927:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176843.257935:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4380 x1872934856304512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176843.257948:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856304512:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786176843.257955:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.257960:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176843.257965:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4380 x1872934856304512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.257997:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176843.258001:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176843.258005:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786176843.258008:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176843.258011:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176843.258013:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786176843.258018:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176843.258021:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b01800.
02000000:00000001:3.0:1786176843.258029:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176843.258032:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176843.258034:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4380.
00000100:00000001:3.0:1786176843.258037:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176843.258041:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258043:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258052:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176843.258053:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.258055:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786176843.258057:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176843.258058:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176843.258061:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa7100 x1872934856304640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258067:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176843.258068:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176843.258070:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176843.258072:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176843.258074:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176843.258077:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786176843.258079:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176843.258081:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176843.258082:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176843.258084:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176843.258086:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786176843.258087:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176843.258090:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176843.258093:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176843.258095:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176843.258098:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856304640, portal 10
00000100:00000001:3.0:1786176843.258099:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176843.258100:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751755008 : -114547957796608 : ffff97d1b9aa7100)
00000100:00000040:3.0:1786176843.258105:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa7100 x1872934856304640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258110:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176843.258111:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176843.258113:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176843.258115:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856304640, offset 0
00000400:00000200:3.0:1786176843.258117:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176843.258120:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176843.258122:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176843.258125:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176843.258126:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176843.258128:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176843.258130:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176843.258132:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254551:254551:256:4294967295] 192.168.202.110@tcp LPNI seq info [254540:254540:8:4294967295]
00000400:00000200:3.0:1786176843.258134:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176843.258139:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176843.258142:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176843.258145:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176843.258147:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176843.258149:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786176843.258153:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176843.258154:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1786176843.258158:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176843.258159:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176843.258161:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176843.258164:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176843.258166:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176843.258167:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176843.258169:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176843.258171:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa7100 x1872934856304640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258177:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176843.258179:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7100 x1872934856304640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258201:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.258202:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176843.258205:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176843.258208:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786176843.258211:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176843.258212:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301739018
00000100:00000001:3.0:1786176843.258215:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301742798
00000100:00000001:3.0:1786176843.258216:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301739018
00000100:00000001:3.0:1786176843.258218:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301742798
00000100:00000001:3.0:1786176843.258219:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176843.258220:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.258222:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176843.258224:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176843/real 1786176843]  req@ffff97d1b9aa7100 x1872934856304640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176843.258229:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176843.258231:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176843.258232:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176843.258233:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176843.258234:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176843.258236:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176843.258238:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa7100 x1872934856304640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176843.258243:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa7100 x1872934856304640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258251:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176843.258252:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258254:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176843.258255:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176843.258257:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa7100 x1872934856304640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258261:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176843.258263:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa7100 x1872934856304640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258267:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176843.258269:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258270:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176843.258270:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258272:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176843.258273:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176843.258274:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176843.258276:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176843.258278:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176843.258280:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa7100 x1872934856304640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176843.258285:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786176843.258288:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176843.258290:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5500 x1872934856304768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258294:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176843.258295:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176843.258297:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176843.258299:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176843.258300:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176843.258302:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786176843.258304:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176843.258305:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176843.258306:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176843.258307:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176843.258310:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786176843.258311:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176843.258313:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176843.258315:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176843.258317:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176843.258319:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856304768, portal 10
00000100:00000001:3.0:1786176843.258321:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176843.258322:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751747840 : -114547957803776 : ffff97d1b9aa5500)
00000100:00000040:3.0:1786176843.258327:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5500 x1872934856304768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258332:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176843.258333:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176843.258335:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176843.258336:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856304768, offset 0
00000400:00000200:3.0:1786176843.258339:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176843.258341:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176843.258343:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176843.258345:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176843.258346:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176843.258348:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176843.258350:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176843.258352:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254552:254552:256:4294967295] 192.168.202.110@tcp LPNI seq info [254541:254541:8:4294967295]
00000400:00000200:3.0:1786176843.258355:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176843.258360:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176843.258363:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176843.258366:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176843.258368:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176843.258370:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786176843.258373:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176843.258375:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1786176843.258378:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176843.258379:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176843.258381:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176843.258383:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176843.258385:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176843.258386:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176843.258388:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176843.258390:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa5500 x1872934856304768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258395:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176843.258398:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5500 x1872934856304768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258402:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.258403:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176843.258404:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176843.258406:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786176843.258408:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176843.258409:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301739143
00000100:00000001:3.0:1786176843.258411:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301742798
00000100:00000001:3.0:1786176843.258413:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301739143
00000100:00000001:3.0:1786176843.258415:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301742798
00000100:00000001:3.0:1786176843.258416:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176843.258417:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.258418:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176843.258420:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176843/real 1786176843]  req@ffff97d1b9aa5500 x1872934856304768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176843.258426:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176843.258427:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176843.258428:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176843.258430:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176843.258431:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176843.258432:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176843.258434:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5500 x1872934856304768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176843.258439:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5500 x1872934856304768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258443:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176843.258444:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258445:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176843.258446:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176843.258448:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5500 x1872934856304768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258453:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176843.258455:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5500 x1872934856304768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258459:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176843.258460:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258461:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176843.258462:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258463:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176843.258464:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176843.258466:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176843.258468:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176843.258469:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176843.258472:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5500 x1872934856304768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176843.258478:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786176843.258480:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176843.258482:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5880 x1872934856304896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258486:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176843.258487:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176843.258488:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176843.258489:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176843.258490:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176843.258492:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786176843.258494:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176843.258495:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176843.258496:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176843.258497:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176843.258499:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786176843.258500:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176843.258502:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176843.258503:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176843.258506:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176843.258507:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856304896, portal 4
00000100:00000001:3.0:1786176843.258526:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176843.258527:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751748736 : -114547957802880 : ffff97d1b9aa5880)
00000100:00000040:3.0:1786176843.258532:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5880 x1872934856304896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258536:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176843.258538:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176843.258540:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176843.258542:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856304896, offset 0
00000400:00000200:3.0:1786176843.258545:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176843.258547:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176843.258550:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176843.258552:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176843.258553:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176843.258555:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176843.258558:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176843.258560:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254553:254553:256:4294967295] 192.168.202.110@tcp LPNI seq info [254542:254542:8:4294967295]
00000400:00000200:3.0:1786176843.258563:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176843.258567:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176843.258570:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176843.258572:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176843.258574:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176843.258576:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786176843.258579:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176843.258581:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1786176843.258584:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176843.258585:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176843.258587:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176843.258589:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176843.258591:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176843.258592:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176843.258594:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176843.258596:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa5880 x1872934856304896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258602:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176843.258604:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5880 x1872934856304896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258608:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.258609:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176843.258611:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176843.258614:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786176843.258616:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176843.258618:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301739019
00000100:00000001:3.0:1786176843.258619:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301742798
00000100:00000001:3.0:1786176843.258621:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301739019
00000100:00000001:3.0:1786176843.258622:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301742798
00000100:00000001:3.0:1786176843.258624:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176843.258625:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.258626:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176843.258628:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176843/real 1786176843]  req@ffff97d1b9aa5880 x1872934856304896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176843.258632:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176843.258633:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176843.258634:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176843.258636:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176843.258637:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176843.258638:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176843.258640:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5880 x1872934856304896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176843.258646:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5880 x1872934856304896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258649:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176843.258650:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258651:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176843.258652:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176843.258654:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5880 x1872934856304896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258658:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176843.258660:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5880 x1872934856304896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258664:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176843.258666:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258667:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176843.258668:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258670:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176843.258671:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176843.258673:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176843.258675:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176843.258676:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176843.258678:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5880 x1872934856304896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176843.258683:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856304896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786176843.258685:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176843.258687:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa6300 x1872934856305024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258694:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176843.258696:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176843.258699:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176843.258700:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176843.258702:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176843.258705:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856305024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786176843.258709:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176843.258710:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176843.258712:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176843.258714:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176843.258717:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786176843.258719:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176843.258723:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176843.258726:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176843.258729:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176843.258732:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856305024, portal 4
00000100:00000001:3.0:1786176843.258735:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176843.258736:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751751424 : -114547957800192 : ffff97d1b9aa6300)
00000100:00000040:3.0:1786176843.258743:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa6300 x1872934856305024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258750:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176843.258752:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176843.258755:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176843.258758:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856305024, offset 0
00000400:00000200:3.0:1786176843.258761:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176843.258763:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176843.258766:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176843.258768:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176843.258770:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176843.258771:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176843.258773:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176843.258775:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254554:254554:256:4294967295] 192.168.202.110@tcp LPNI seq info [254543:254543:8:4294967295]
00000400:00000200:3.0:1786176843.258778:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176843.258782:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176843.258785:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176843.258788:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176843.258790:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176843.258792:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786176843.258794:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176843.258797:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1786176843.258799:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176843.258801:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176843.258803:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176843.258805:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176843.258807:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176843.258808:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176843.258810:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176843.258812:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa6300 x1872934856305024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258822:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176843.258824:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa6300 x1872934856305024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258829:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.258830:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176843.258831:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176843.258834:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786176843.258836:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176843.258837:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301739027
00000100:00000001:3.0:1786176843.258839:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301742798
00000100:00000001:3.0:1786176843.258840:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301739027
00000100:00000001:3.0:1786176843.258842:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301742798
00000100:00000001:3.0:1786176843.258843:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176843.258845:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.258846:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176843.258850:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176843/real 1786176843]  req@ffff97d1b9aa6300 x1872934856305024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176843.258857:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176843.258859:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176843.258861:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176843.258863:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176843.258865:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176843.258868:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176843.258871:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa6300 x1872934856305024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176843.258880:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa6300 x1872934856305024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258887:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176843.258889:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258891:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176843.258892:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176843.258896:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa6300 x1872934856305024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258903:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176843.258907:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa6300 x1872934856305024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258915:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176843.258918:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258920:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176843.258921:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258923:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176843.258925:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176843.258927:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176843.258931:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176843.258934:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176843.258939:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa6300 x1872934856305024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176843.258948:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856305024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786176843.258952:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258955:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176843.258959:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa7100 x1872934856304640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.258966:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176843.258968:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176843.258971:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786176843.258974:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176843.258975:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176843.258978:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786176843.258980:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176843.258982:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03000.
02000000:00000001:3.0:1786176843.258985:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176843.258986:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176843.258988:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa7100.
00000100:00000001:3.0:1786176843.258991:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176843.258993:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.258994:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176843.258998:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5500 x1872934856304768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.259006:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176843.259007:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176843.259009:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786176843.259011:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176843.259012:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176843.259014:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786176843.259016:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176843.259017:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03400.
02000000:00000001:3.0:1786176843.259019:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176843.259020:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176843.259022:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5500.
00000100:00000001:3.0:1786176843.259024:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176843.259025:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.259027:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176843.259030:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5880 x1872934856304896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.259036:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176843.259037:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176843.259039:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786176843.259041:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176843.259042:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176843.259044:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786176843.259046:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176843.259048:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00000.
02000000:00000001:3.0:1786176843.259050:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176843.259052:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176843.259053:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5880.
00000100:00000001:3.0:1786176843.259055:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176843.259057:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.259059:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176843.259063:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa6300 x1872934856305024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176843.259069:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176843.259070:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176843.259071:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786176843.259073:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176843.259074:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176843.259076:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786176843.259078:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176843.259080:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00c00.
02000000:00000001:3.0:1786176843.259084:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176843.259085:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176843.259086:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa6300.
00000100:00000001:3.0:1786176843.259089:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176843.259091:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.259093:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176843.259098:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176843.259099:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176843.259102:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786176843.259104:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176843.832795:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176843.832803:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176843.832809:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176844.280605:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176844.280613:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176844.280615:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176844.280621:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176844.280624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176844.280629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176844.280632:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176844.280635:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176844.280639:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176844.856695:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176844.856699:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176844.856702:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176845.304640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176845.304646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176845.304648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176845.304652:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176845.304655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176845.304659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176845.304662:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176845.304670:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176845.304672:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176845.880656:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176845.880659:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176845.880662:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176846.328610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176846.328617:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176846.328619:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176846.328624:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176846.328627:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176846.328630:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176846.328632:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176846.328634:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176846.328636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176846.904703:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176846.904706:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176846.904709:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176847.096630:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301742798, at 4301742848
00000100:00000010:0.0:1786176847.096648:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad374e0.
00000100:00000001:2.0:1786176847.352619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176847.352632:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176847.352633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176847.352638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176847.352641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176847.352644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176847.352647:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176847.352649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176847.352651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176847.928676:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176847.928680:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176847.928683:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176848.376593:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176848.376602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176848.376605:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176848.376610:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176848.376614:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176848.376617:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176848.376620:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176848.376623:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176848.376629:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.376850:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786176848.376857:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176848.376862:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176848.376873:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa6680 x1872934856305152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.376895:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176848.376900:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176848.376908:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176848.376913:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176848.376919:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176848.376930:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856305152:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786176848.376937:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176848.376940:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176848.376945:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176848.376950:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176848.376957:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786176848.376963:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176848.376973:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176848.376981:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176848.376993:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176848.377004:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856305152, portal 25
00000100:00000001:3.0:1786176848.377011:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176848.377015:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751752320 : -114547957799296 : ffff97d1b9aa6680)
00000100:00000040:3.0:1786176848.377028:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa6680 x1872934856305152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.377042:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176848.377048:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176848.377052:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176848.377058:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856305152, offset 0
00000400:00000200:3.0:1786176848.377073:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176848.377086:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176848.377094:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176848.377102:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176848.377109:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176848.377116:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176848.377121:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176848.377128:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254555:254555:256:4294967295] 192.168.202.110@tcp LPNI seq info [254544:254544:8:4294967295]
00000400:00000200:3.0:1786176848.377149:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176848.377165:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176848.377179:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176848.377186:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176848.377195:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176848.377200:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176848.377210:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176848.377214:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786176848.377221:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176848.377224:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176848.377229:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176848.377239:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176848.377244:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176848.377247:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176848.377254:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176848.377261:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa6680 x1872934856305152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.377281:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176848.377290:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa6680 x1872934856305152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.377313:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.377318:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176848.377329:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176848.377338:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176848.377345:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301747917
00000100:00000010:3.0:1786176848.377355:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad379c0.
00000100:00000001:3.0:1786176848.377366:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786176848.377376:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301747917
00000100:00000010:3.0:1786176848.377385:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37120.
00000100:00000001:3.0:1786176848.377391:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786176848.377396:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176848.377400:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.377405:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176848.377409:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176848/real 1786176848]  req@ffff97d1b9aa6680 x1872934856305152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176848.377444:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176848.377447:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176848.377450:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176848.377455:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176848.377458:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176848.377463:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176848.377470:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa6680 x1872934856305152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176848.377486:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa6680 x1872934856305152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.377502:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176848.377505:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176848.377533:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176848.377536:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786176848.377537:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786176848.377538:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:3.0:1786176848.377539:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176848.377543:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1786176848.377545:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00100000:3.0:1786176848.377548:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa6680 x1872934856305152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1786176848.377550:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:1.0:1786176848.377557:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:3.0:1786176848.377558:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1786176848.377563:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786176848.377565:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa6680 x1872934856305152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786176848.377572:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176848.377575:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176848.377578:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786176848.377579:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.377583:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176848.377585:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176848.377586:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176848.377588:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176848.377593:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786176848.377593:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176848.377599:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000020:00000040:1.0:1786176848.377600:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786176848.377605:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00080000:1.0:1786176848.377606:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176848.377608:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:1.0:1786176848.377610:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176848.377613:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1786176848.377614:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa6680 x1872934856305152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786176848.377616:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176848.377622:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176848.377626:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786176848.377629:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00100000:3.0:1786176848.377630:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856305152:192.168.202.110@tcp:250:kworker.0
00000008:00000001:1.0:1786176848.377633:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176848.377636:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.377637:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:1.0:1786176848.377640:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa4700.
00000100:00000001:3.0:1786176848.377640:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176848.377644:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa6680 x1872934856305152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1786176848.377645:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:1.0:1786176848.377650:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786176848.377653:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176848.377657:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786176848.377659:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786176848.377659:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176848.377663:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000001:1.0:1786176848.377664:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176848.377667:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000010:1.0:1786176848.377670:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:3.0:1786176848.377671:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176848.377674:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000001:1.0:1786176848.377675:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786176848.377676:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786176848.377680:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176848.377683:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03800.
00000100:00100000:1.0:1786176848.377687:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1786176848.377687:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176848.377690:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176848.377692:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa6680.
00000100:00000001:3.0:1786176848.377697:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176848.377699:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.377704:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.377713:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176848.377715:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176848.377716:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176848.377721:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.377722:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786176848.377727:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1786176848.377731:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c2491d39] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.377731:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176848.377739:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5c00 x1872934856305280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176848.377748:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.377756:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:1.0:1786176848.377757:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7081)
00000100:00000001:3.0:1786176848.377758:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176848.377763:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176848.377765:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176848.377767:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176848.377771:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856305280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786176848.377776:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176848.377779:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176848.377780:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176848.377782:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176848.377786:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786176848.377789:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176848.377793:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176848.377797:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176848.377801:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176848.377804:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856305280, portal 10
00000100:00000001:3.0:1786176848.377806:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176848.377808:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751749632 : -114547957801984 : ffff97d1b9aa5c00)
00000100:00000040:3.0:1786176848.377816:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5c00 x1872934856305280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.377821:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176848.377823:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176848.377825:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176848.377828:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856305280, offset 0
00000400:00000200:3.0:1786176848.377831:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176848.377834:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176848.377837:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176848.377839:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176848.377842:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176848.377843:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176848.377845:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176848.377847:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254556:254556:256:4294967295] 192.168.202.110@tcp LPNI seq info [254545:254545:8:4294967295]
00000400:00000200:3.0:1786176848.377851:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176848.377856:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176848.377859:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176848.377861:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176848.377864:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176848.377867:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176848.377870:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176848.377872:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786176848.377875:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176848.377877:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176848.377879:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176848.377881:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176848.377884:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176848.377885:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176848.377887:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176848.377889:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa5c00 x1872934856305280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.377895:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176848.377897:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5c00 x1872934856305280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.377902:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.377903:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176848.377906:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176848.377909:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786176848.377912:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176848.377913:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301744138
00000100:00000001:3.0:1786176848.377915:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301747917
00000100:00000001:3.0:1786176848.377917:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301744138
00000100:00000001:3.0:1786176848.377919:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301747917
00000100:00000001:3.0:1786176848.377920:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176848.377921:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.377923:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176848.377925:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176848/real 1786176848]  req@ffff97d1b9aa5c00 x1872934856305280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176848.377930:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176848.377931:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176848.377933:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176848.377934:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176848.377935:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176848.377937:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176848.377939:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5c00 x1872934856305280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176848.377944:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5c00 x1872934856305280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.377948:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176848.377949:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.377950:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176848.377951:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176848.377953:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5c00 x1872934856305280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.377957:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176848.377960:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5c00 x1872934856305280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.377964:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176848.377965:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.377966:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176848.377967:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.377968:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176848.377969:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176848.377972:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176848.377975:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176848.377977:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176848.377981:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5c00 x1872934856305280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176848.377988:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856305280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786176848.377992:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176848.377996:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa7800 x1872934856305408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378004:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176848.378005:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176848.378008:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176848.378010:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176848.378012:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176848.378015:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856305408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786176848.378019:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176848.378021:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176848.378022:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176848.378024:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176848.378027:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786176848.378028:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176848.378031:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176848.378032:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176848.378035:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176848.378037:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856305408, portal 10
00000100:00000001:3.0:1786176848.378038:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176848.378039:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751756800 : -114547957794816 : ffff97d1b9aa7800)
00000100:00000040:3.0:1786176848.378044:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa7800 x1872934856305408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378048:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176848.378049:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176848.378051:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176848.378053:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856305408, offset 0
00000400:00000200:3.0:1786176848.378055:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176848.378058:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176848.378060:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176848.378062:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176848.378064:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176848.378066:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176848.378067:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176848.378070:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254557:254557:256:4294967295] 192.168.202.110@tcp LPNI seq info [254546:254546:8:4294967295]
00000400:00000200:3.0:1786176848.378072:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176848.378077:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176848.378079:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176848.378081:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176848.378084:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176848.378086:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176848.378089:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176848.378090:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786176848.378093:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176848.378094:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176848.378096:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176848.378098:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176848.378100:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176848.378102:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176848.378103:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176848.378106:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa7800 x1872934856305408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378111:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176848.378113:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7800 x1872934856305408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378117:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.378118:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176848.378120:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176848.378122:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786176848.378124:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176848.378125:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301744263
00000100:00000001:3.0:1786176848.378127:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301747917
00000100:00000001:3.0:1786176848.378128:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301744263
00000100:00000001:3.0:1786176848.378130:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301747917
00000100:00000001:3.0:1786176848.378146:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176848.378148:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.378149:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176848.378152:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176848/real 1786176848]  req@ffff97d1b9aa7800 x1872934856305408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176848.378157:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176848.378159:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176848.378160:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176848.378161:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176848.378163:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176848.378164:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176848.378166:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa7800 x1872934856305408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176848.378172:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa7800 x1872934856305408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378176:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176848.378177:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378178:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176848.378179:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176848.378181:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa7800 x1872934856305408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378185:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176848.378187:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa7800 x1872934856305408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378191:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176848.378193:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378193:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176848.378194:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378196:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176848.378197:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176848.378199:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176848.378201:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176848.378203:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176848.378205:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa7800 x1872934856305408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176848.378210:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856305408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786176848.378213:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176848.378215:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5f80 x1872934856305536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378219:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176848.378220:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176848.378221:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176848.378223:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176848.378224:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176848.378225:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856305536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786176848.378227:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176848.378228:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176848.378229:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176848.378231:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176848.378233:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786176848.378235:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176848.378237:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176848.378239:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176848.378241:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176848.378243:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856305536, portal 4
00000100:00000001:3.0:1786176848.378244:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176848.378245:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751750528 : -114547957801088 : ffff97d1b9aa5f80)
00000100:00000040:3.0:1786176848.378249:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5f80 x1872934856305536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378254:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176848.378255:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176848.378256:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176848.378258:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856305536, offset 0
00000400:00000200:3.0:1786176848.378260:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176848.378263:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176848.378265:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176848.378267:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176848.378269:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176848.378270:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176848.378272:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176848.378275:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254558:254558:256:4294967295] 192.168.202.110@tcp LPNI seq info [254547:254547:8:4294967295]
00000400:00000200:3.0:1786176848.378279:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176848.378285:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176848.378290:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176848.378293:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176848.378297:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176848.378300:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176848.378304:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176848.378307:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786176848.378311:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176848.378313:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176848.378316:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176848.378320:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176848.378323:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176848.378325:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176848.378328:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176848.378331:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa5f80 x1872934856305536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378339:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176848.378342:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5f80 x1872934856305536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378355:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.378357:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176848.378361:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176848.378364:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786176848.378368:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176848.378370:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301744138
00000100:00000001:3.0:1786176848.378373:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301747917
00000100:00000001:3.0:1786176848.378375:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301744138
00000100:00000001:3.0:1786176848.378378:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301747917
00000100:00000001:3.0:1786176848.378380:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176848.378382:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.378384:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176848.378387:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176848/real 1786176848]  req@ffff97d1b9aa5f80 x1872934856305536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176848.378392:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176848.378393:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176848.378395:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176848.378396:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176848.378398:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176848.378399:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176848.378401:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5f80 x1872934856305536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176848.378407:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5f80 x1872934856305536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378411:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176848.378412:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378413:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176848.378413:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176848.378416:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5f80 x1872934856305536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378421:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176848.378423:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5f80 x1872934856305536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378427:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176848.378429:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378430:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176848.378431:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378432:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176848.378433:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176848.378434:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176848.378436:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176848.378437:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176848.378440:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5f80 x1872934856305536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176848.378444:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856305536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786176848.378447:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378448:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176848.378450:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5c00 x1872934856305280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378454:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176848.378455:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176848.378456:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786176848.378458:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176848.378459:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176848.378460:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786176848.378462:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176848.378463:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03c00.
02000000:00000001:3.0:1786176848.378465:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176848.378466:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176848.378467:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5c00.
00000100:00000001:3.0:1786176848.378469:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176848.378470:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378471:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176848.378473:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa7800 x1872934856305408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378477:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176848.378478:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176848.378479:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786176848.378480:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176848.378481:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176848.378481:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786176848.378483:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176848.378483:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b02400.
02000000:00000001:3.0:1786176848.378485:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176848.378485:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176848.378486:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa7800.
00000100:00000001:3.0:1786176848.378488:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176848.378488:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378489:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176848.378491:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5f80 x1872934856305536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378495:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176848.378495:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176848.378496:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786176848.378497:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176848.378498:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176848.378499:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786176848.378500:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176848.378501:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b02c00.
02000000:00000001:3.0:1786176848.378502:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176848.378502:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176848.378503:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5f80.
00000100:00000001:3.0:1786176848.378504:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176848.378506:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378507:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378525:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176848.378526:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.378527:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786176848.378528:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176848.378529:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176848.378532:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378536:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176848.378538:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176848.378539:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176848.378540:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176848.378542:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176848.378543:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856305664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786176848.378546:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176848.378547:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176848.378549:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176848.378550:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176848.378553:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786176848.378555:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176848.378559:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786176848.378562:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786176848.378565:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786176848.378568:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856305664, portal 4
00000100:00000001:3.0:1786176848.378570:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176848.378571:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751744256 : -114547957807360 : ffff97d1b9aa4700)
00000100:00000040:3.0:1786176848.378578:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378585:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176848.378588:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786176848.378590:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786176848.378593:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856305664, offset 0
00000400:00000200:3.0:1786176848.378597:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176848.378602:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176848.378606:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786176848.378609:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786176848.378611:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786176848.378613:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786176848.378614:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176848.378616:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254559:254559:256:4294967295] 192.168.202.110@tcp LPNI seq info [254548:254548:8:4294967295]
00000400:00000200:3.0:1786176848.378619:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176848.378624:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786176848.378628:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176848.378630:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786176848.378633:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786176848.378635:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786176848.378638:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786176848.378640:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786176848.378642:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786176848.378644:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786176848.378646:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786176848.378648:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786176848.378651:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786176848.378652:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786176848.378653:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786176848.378656:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378661:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176848.378663:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378668:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.378669:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176848.378671:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176848.378673:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786176848.378675:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176848.378677:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301744147
00000100:00000001:3.0:1786176848.378679:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301747917
00000100:00000001:3.0:1786176848.378680:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301744147
00000100:00000001:3.0:1786176848.378682:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301747917
00000100:00000001:3.0:1786176848.378683:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176848.378684:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.378685:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786176848.378687:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176848/real 1786176848]  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786176848.378692:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786176848.378693:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786176848.378694:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786176848.378696:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786176848.378697:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786176848.378698:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786176848.378700:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786176848.378707:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378714:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176848.378715:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378717:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176848.378718:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176848.378723:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378731:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176848.378735:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378743:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176848.378746:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378748:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176848.378749:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378751:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786176848.378753:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176848.378756:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176848.378758:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176848.378760:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176848.378763:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176848.378769:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856305664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786176848.378773:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378775:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176848.378778:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4700 x1872934856305664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176848.378784:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176848.378786:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786176848.378788:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786176848.378791:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786176848.378792:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786176848.378794:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786176848.378796:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786176848.378798:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00c00.
02000000:00000001:3.0:1786176848.378801:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176848.378802:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786176848.378804:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4700.
00000100:00000001:3.0:1786176848.378806:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786176848.378808:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378810:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176848.378813:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176848.378814:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176848.378816:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786176848.378818:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176848.952620:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176848.952626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176848.952630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176848.952631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176848.952634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176848.952635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176848.952683:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176848.952687:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176848.952689:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176849.400636:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176849.400644:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176849.400646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176849.400651:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176849.400654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176849.400657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176849.400660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176849.400662:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176849.400664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176849.976625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176849.976633:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176849.976636:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176849.976639:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176849.976644:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176849.976647:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176849.976650:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176849.976652:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176849.976656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176849.976658:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176849.976659:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176849.976661:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176849.976663:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176849.976665:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176849.976666:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176849.976668:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176849.976673:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176849.976674:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176849.976732:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176849.976736:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176849.976739:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176850.424575:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176850.424581:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176850.424583:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176850.424587:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176850.424590:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176850.424593:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176850.424595:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176850.424598:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176850.424600:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176851.000678:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176851.000686:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176851.000692:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176851.000694:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176851.000697:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176851.000699:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176851.000782:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176851.000788:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176851.000792:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176851.448643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176851.448652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176851.448655:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176851.448662:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176851.448666:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176851.448677:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176851.448681:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176851.448684:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176851.448688:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176852.024648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176852.024654:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176852.024658:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176852.024659:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176852.024662:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176852.024663:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176852.024714:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176852.024718:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176852.024721:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176852.216626:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301747917, at 4301747968
00000100:00000040:1.0:1786176852.216626:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301747917, at 4301747968
00000100:00000010:0.0:1786176852.216640:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad379c0.
00000100:00000010:1.0:1786176852.216643:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37120.
00000100:00000001:2.0:1786176852.472639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176852.472646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176852.472648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176852.472653:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176852.472656:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176852.472659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176852.472661:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176852.472663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176852.472666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176853.048587:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176853.048593:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176853.048598:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176853.048599:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176853.048602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176853.048604:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176853.048606:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176853.048608:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176853.048611:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176853.048613:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176853.048614:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176853.048615:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176853.048706:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176853.048709:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176853.048712:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176853.496603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176853.496609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:3.0:1786176853.496610:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786176853.496611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176853.496618:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:3.0:1786176853.496619:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786176853.496622:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1786176853.496623:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1786176853.496625:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176853.496627:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176853.496630:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786176853.496631:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176853.496634:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176853.496637:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7076
00000100:00000001:2.0:1786176853.496638:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176853.496648:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176853.496654:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176853.496658:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176853.496662:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.496666:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176853.496669:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.496672:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176853.496676:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176853.496678:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176853.496681:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176853.496684:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176853.496687:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176853.496690:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176853.496694:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.496696:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176853.496698:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.496701:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176853.496704:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176853.496708:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176853.496711:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176853.496714:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176853.496716:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176853.496718:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176853.496720:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176853.496722:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176853.496729:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f800.
00000020:00000040:3.0:1786176853.496732:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176853.496735:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176853.496737:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176853.496739:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176853.496742:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176853.496745:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176853.496752:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786176853.496757:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176853.496765:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176853.496787:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176853.496792:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176853.496798:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4d339d9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176853.496822:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176853.496825:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176853.496828:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176853.496829:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176853.496837:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176853.496840:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176853.496843:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7076
00000100:00000001:1.0:1786176853.496844:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176853.496848:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000400:00000200:3.0:1786176853.496848:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176853.496853:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000400:00000200:3.0:1786176853.496855:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176853.496869:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786176853.496875:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176853.496875:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.496880:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176853.496883:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.496887:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176853.496891:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176853.496893:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786176853.496896:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176853.496900:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000040:3.0:1786176853.496900:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:1.0:1786176853.496905:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176853.496905:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176853.496911:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:1.0:1786176853.496914:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1786176853.496914:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.496918:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176853.496919:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.496920:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176853.496922:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786176853.496924:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176853.496925:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176853.496926:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786176853.496927:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176853.496931:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176853.496931:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176853.496935:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176853.496936:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856305792:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786176853.496938:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176853.496943:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:3.0:1786176853.496944:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176853.496947:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176853.496948:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1786176853.496949:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176853.496951:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176853.496954:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69fb80.
02000000:00000001:1.0:1786176853.496955:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:3.0:1786176853.496958:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176853.496960:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000010:1.0:1786176853.496962:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
02000000:00000001:3.0:1786176853.496962:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176853.496964:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786176853.496966:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176853.496967:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176853.496970:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176853.496974:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:3.0:1786176853.496976:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000400:00000010:1.0:1786176853.496978:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00000001:3.0:1786176853.496980:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176853.496985:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00100000:3.0:1786176853.496989:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1786176853.496994:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856305792, portal 25
00000100:00000001:3.0:1786176853.496997:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176853.496998:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176853.497001:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
00000100:00000001:3.0:1786176853.497002:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176853.497012:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176853.497012:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014f55bbb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.497037:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786176853.497040:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176853.497043:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786176853.497045:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1786176853.497047:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00080000:3.0:1786176853.497048:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000200:1.0:1786176853.497051:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856305792, offset 0
00000100:00000001:3.0:1786176853.497051:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176853.497054:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176853.497057:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786176853.497061:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786176853.497064:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7076
00000400:00000200:3.0:1786176853.497068:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176853.497072:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176853.497073:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176853.497077:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176853.497079:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176853.497096:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176853.497104:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1786176853.497106:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176853.497110:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:3.0:1786176853.497113:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.497116:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176853.497117:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:3.0:1786176853.497119:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:1.0:1786176853.497122:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000040:3.0:1786176853.497123:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786176853.497126:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786176853.497129:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254560:254560:256:4294967295] 192.168.202.110@tcp LPNI seq info [254549:254549:8:4294967295]
00000020:00000040:3.0:1786176853.497130:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786176853.497134:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786176853.497138:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1786176853.497138:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786176853.497144:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.497146:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176853.497152:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176853.497155:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786176853.497162:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:1.0:1786176853.497163:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:3.0:1786176853.497166:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786176853.497170:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000010:1.0:1786176853.497172:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000100:00000001:3.0:1786176853.497173:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176853.497177:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176853.497180:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176853.497183:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176853.497186:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786176853.497186:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176853.497189:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176853.497192:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176853.497193:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:3.0:1786176853.497194:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176853.497199:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ca80.
00000020:00000040:3.0:1786176853.497203:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000800:00000200:1.1:1786176853.497204:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:3.0:1786176853.497206:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176853.497208:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786176853.497211:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:3.0:1786176853.497212:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176853.497214:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176853.497216:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176853.497221:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000800:00000100:1.0:1786176853.497222:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176853.497227:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786176853.497228:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000100:00100000:3.0:1786176853.497234:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176853.497238:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1786176853.497241:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176853.497244:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176853.497247:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ffb0865] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176853.497262:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1786176853.497263:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176853.497267:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786176853.497270:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000100:00080000:3.0:1786176853.497270:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000010:1.0:1786176853.497275:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:3.0:1786176853.497275:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176853.497279:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176853.497282:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1786176853.497283:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176853.497287:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7076
00000100:00000200:1.0:1786176853.497289:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786176853.497294:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176853.497301:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176853.497305:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176853.497307:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176853.497309:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.497312:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786176853.497314:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176853.497317:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.497323:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176853.497327:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176853.497330:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:1.0:1786176853.497331:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176853.497332:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176853.497335:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:3.0:1786176853.497336:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786176853.497339:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786176853.497341:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176853.497342:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176853.497345:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.497348:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176853.497349:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176853.497364:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301753037
00000100:00000040:3.0:1786176853.497365:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176853.497370:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176853.497372:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000010:1.0:1786176853.497373:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1ae0.
00000100:00080000:3.0:1786176853.497374:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176853.497378:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176853.497379:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176853.497381:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:3.0:1786176853.497382:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176853.497386:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176853.497387:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176853.497389:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301753037
00000008:00100000:3.0:1786176853.497390:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000010:1.0:1786176853.497393:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b10c0.
00000008:00000001:3.0:1786176853.497394:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176853.497398:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.497399:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000010:3.0:1786176853.497402:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f480.
00000100:00000001:1.0:1786176853.497403:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1786176853.497408:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176853.497409:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.497414:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000001:3.0:1786176853.497414:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176853.497417:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176853.497420:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:1.0:1786176853.497421:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176853/real 1786176853]  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176853.497422:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176853.497425:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176853.497431:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000400:00000001:1.0:1786176853.497434:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1786176853.497435:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176853.497436:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176853.497439:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000100:00100000:3.0:1786176853.497442:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176853.497444:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176853.497447:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176853.497452:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:3.0:1786176853.497456:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1786176853.497458:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176853.497460:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176853.497469:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092aa1aa7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176853.497473:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176853.497491:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.497493:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00080000:3.0:1786176853.497494:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176853.497495:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.497500:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:3.0:1786176853.497501:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1786176853.497503:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176853.497507:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.497532:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176853.497535:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.497540:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176853.497543:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.497544:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176853.497545:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.497547:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176853.497549:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176853.497551:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786176853.497552:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176853.497557:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00080000:1.0:1786176853.497558:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176853.497563:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1786176853.497565:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:1.0:1786176853.497569:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176853.497573:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7076
00000100:00100000:1.0:1786176853.497584:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856305792:192.168.202.110@tcp:250:kworker.0
00000400:00000200:3.0:1786176853.497585:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176853.497590:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.497593:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000400:00000200:3.0:1786176853.497596:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:1.0:1786176853.497599:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856305792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176853.497602:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176853.497612:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176853.497615:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000001:3.0:1786176853.497617:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1786176853.497618:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176853.497622:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
00000100:00000040:3.0:1786176853.497624:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176853.497625:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176853.497629:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000001:3.0:1786176853.497629:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786176853.497631:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786176853.497634:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000001:1.0:1786176853.497635:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000020:00000040:3.0:1786176853.497637:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
02000000:00000010:1.0:1786176853.497638:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
00000020:00000001:3.0:1786176853.497641:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
02000000:00000001:1.0:1786176853.497642:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000020:00000040:3.0:1786176853.497644:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
02000000:00000001:1.0:1786176853.497645:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000020:00000001:3.0:1786176853.497646:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000010:1.0:1786176853.497648:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000020:00000001:3.0:1786176853.497650:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176853.497652:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176853.497655:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176853.497656:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176853.497658:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176853.497663:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176853.497667:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000040:3.0:1786176853.497668:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176853.497670:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.497674:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786176853.497674:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176853.497676:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176853.497679:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1786176853.497681:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000040:1.0:1786176853.497683:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176853.497690:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176853.497692:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176853.497695:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176853.497697:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176853.497698:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176853.497701:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176853.497703:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786176853.497704:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176853.497708:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856305920:192.168.202.110@tcp:38:kworker.0
02000000:00000001:3.0:1786176853.497709:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176853.497712:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176853.497714:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176853.497715:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1786176853.497717:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786176853.497718:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1786176853.497720:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000001:1.0:1786176853.497721:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176853.497725:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
00000008:00000001:3.0:1786176853.497725:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176853.497731:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176853.497731:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176853.497737:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:3.0:1786176853.497738:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f100.
00000400:00000010:1.0:1786176853.497742:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000020:00000040:3.0:1786176853.497742:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000400:00000010:1.0:1786176853.497749:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
02000000:00000001:3.0:1786176853.497749:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176853.497752:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176853.497754:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856305920, portal 10
00000100:00000001:3.0:1786176853.497755:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176853.497758:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176853.497759:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176853.497762:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
02000000:00000001:3.0:1786176853.497762:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176853.497773:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1786176853.497773:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:1.0:1786176853.497790:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786176853.497792:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176853.497793:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176853.497796:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000200:1.0:1786176853.497799:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856305920, offset 0
00000400:00000200:1.0:1786176853.497804:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00100000:3.0:1786176853.497804:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176853.497811:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176853.497817:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176853.497824:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1786176853.497829:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176853.497830:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1786176853.497834:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176853.497835:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176853.497842:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:3.0:1786176853.497843:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1ef9f45] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176853.497845:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254561:254561:256:4294967295] 192.168.202.110@tcp LPNI seq info [254550:254550:8:4294967295]
00000400:00000200:1.0:1786176853.497856:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176853.497873:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786176853.497874:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176853.497881:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7086)
00000800:00000200:1.0:1786176853.497882:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176853.497888:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176853.497893:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176853.497897:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786176853.497903:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176853.497906:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786176853.497911:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176853.497913:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176853.497916:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176853.497920:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176853.497924:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:1.0:1786176853.497936:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176853.497939:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176853.497942:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.497948:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176853.497950:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.497955:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.497956:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176853.497959:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176853.497962:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176853.497964:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176853.497965:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301749258
00000100:00000001:1.0:1786176853.497968:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301753037
00000100:00000001:1.0:1786176853.497969:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301749258
00000100:00000001:1.0:1786176853.497970:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301753037
00000100:00000001:1.0:1786176853.497972:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176853.497973:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.497975:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176853.497977:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176853/real 1786176853]  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176853.497982:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176853.497983:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176853.497984:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176853.497985:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176853.497987:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176853.497988:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176853.497990:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176853.497995:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498003:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176853.498004:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498006:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176853.498007:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176853.498009:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498013:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176853.498015:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498019:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176853.498021:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498022:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176853.498023:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498024:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176853.498025:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176853.498027:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176853.498029:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176853.498030:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176853.498033:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176853.498038:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856305920:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176853.498041:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176853.498043:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498047:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176853.498048:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176853.498049:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176853.498051:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176853.498052:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176853.498054:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176853.498056:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176853.498057:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176853.498058:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176853.498059:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176853.498062:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176853.498063:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176853.498065:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176853.498067:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176853.498069:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176853.498071:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856306048, portal 10
00000100:00000001:1.0:1786176853.498072:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176853.498073:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:1.0:1786176853.498077:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498081:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176853.498097:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176853.498099:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000200:1.0:1786176853.498101:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856306048, offset 0
00000400:00000200:1.0:1786176853.498103:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176853.498106:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176853.498109:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176853.498111:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176853.498113:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176853.498115:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176853.498116:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176853.498128:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254562:254562:256:4294967295] 192.168.202.110@tcp LPNI seq info [254551:254551:8:4294967295]
00000400:00000200:1.0:1786176853.498131:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176853.498136:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176853.498139:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176853.498141:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176853.498144:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176853.498146:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786176853.498149:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176853.498151:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786176853.498154:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176853.498155:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176853.498157:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176853.498161:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176853.498164:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:1.0:1786176853.498166:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176853.498168:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176853.498172:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498180:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176853.498184:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498191:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.498193:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176853.498196:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176853.498199:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176853.498203:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176853.498205:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301749383
00000100:00000001:1.0:1786176853.498208:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301753037
00000100:00000001:1.0:1786176853.498210:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301749383
00000100:00000001:1.0:1786176853.498212:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301753037
00000100:00000001:1.0:1786176853.498214:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176853.498217:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.498219:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176853.498223:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176853/real 1786176853]  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176853.498243:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176853.498244:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176853.498246:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176853.498249:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176853.498251:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176853.498253:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176853.498257:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176853.498267:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498271:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176853.498273:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498274:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176853.498275:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176853.498278:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498282:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176853.498285:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498289:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176853.498291:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498292:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176853.498292:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498293:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176853.498295:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176853.498296:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176853.498298:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176853.498299:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176853.498303:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176853.498308:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176853.498311:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176853.498313:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498317:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176853.498318:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176853.498319:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176853.498320:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176853.498322:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176853.498323:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176853.498326:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176853.498327:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176853.498328:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176853.498329:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176853.498332:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176853.498333:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176853.498336:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176853.498338:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176853.498340:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176853.498342:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856306176, portal 4
00000100:00000001:1.0:1786176853.498343:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176853.498345:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:1.0:1786176853.498349:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498363:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176853.498364:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176853.498365:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000200:1.0:1786176853.498367:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856306176, offset 0
00000400:00000200:1.0:1786176853.498370:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176853.498373:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176853.498375:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176853.498377:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176853.498379:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176853.498381:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176853.498383:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176853.498385:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254563:254563:256:4294967295] 192.168.202.110@tcp LPNI seq info [254552:254552:8:4294967295]
00000400:00000200:1.0:1786176853.498387:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176853.498392:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176853.498394:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176853.498396:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176853.498399:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176853.498401:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786176853.498404:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176853.498406:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786176853.498409:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176853.498410:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176853.498412:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176853.498414:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176853.498417:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:1.0:1786176853.498418:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176853.498419:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176853.498421:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498426:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176853.498428:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498432:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.498433:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176853.498435:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176853.498437:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176853.498439:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176853.498441:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301749258
00000100:00000001:1.0:1786176853.498443:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301753037
00000100:00000001:1.0:1786176853.498444:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301749258
00000100:00000001:1.0:1786176853.498446:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301753037
00000100:00000001:1.0:1786176853.498447:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176853.498448:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.498449:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176853.498451:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176853/real 1786176853]  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176853.498456:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176853.498457:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176853.498458:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176853.498459:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176853.498460:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176853.498462:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176853.498464:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176853.498469:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498473:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176853.498473:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498474:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176853.498475:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176853.498477:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498481:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176853.498483:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498487:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176853.498489:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498490:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176853.498491:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498492:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176853.498493:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176853.498495:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176853.498497:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176853.498498:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176853.498500:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176853.498505:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176853.498507:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498524:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176853.498527:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856305920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498531:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176853.498532:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176853.498533:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176853.498535:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176853.498536:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176853.498537:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176853.498538:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176853.498539:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:1.0:1786176853.498541:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176853.498542:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176853.498543:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:1.0:1786176853.498545:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176853.498546:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498547:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176853.498549:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856306048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498557:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176853.498558:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176853.498559:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176853.498561:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176853.498561:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176853.498562:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176853.498563:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176853.498564:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176853.498565:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176853.498566:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176853.498567:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:1.0:1786176853.498568:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176853.498569:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498569:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176853.498571:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856306176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498575:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176853.498576:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176853.498577:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176853.498578:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176853.498578:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176853.498579:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176853.498580:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176853.498581:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786176853.498583:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176853.498583:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176853.498584:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:1.0:1786176853.498585:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176853.498586:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498587:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498590:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176853.498590:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.498592:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176853.498592:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176853.498593:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176853.498596:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498600:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176853.498601:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176853.498603:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176853.498604:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176853.498605:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176853.498607:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176853.498609:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176853.498610:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176853.498611:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176853.498613:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176853.498615:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176853.498616:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176853.498618:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176853.498620:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176853.498622:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176853.498624:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856306304, portal 4
00000100:00000001:1.0:1786176853.498625:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176853.498626:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:1.0:1786176853.498630:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498635:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176853.498636:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176853.498638:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000200:1.0:1786176853.498639:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856306304, offset 0
00000400:00000200:1.0:1786176853.498642:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176853.498644:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176853.498646:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176853.498648:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176853.498649:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176853.498651:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176853.498652:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176853.498655:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254564:254564:256:4294967295] 192.168.202.110@tcp LPNI seq info [254553:254553:8:4294967295]
00000400:00000200:1.0:1786176853.498658:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176853.498662:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176853.498665:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176853.498667:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176853.498669:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176853.498671:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786176853.498673:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176853.498675:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786176853.498678:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176853.498679:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176853.498681:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176853.498683:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176853.498685:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:1.0:1786176853.498686:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176853.498688:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176853.498690:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498695:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176853.498697:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176878 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498701:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.498702:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176853.498704:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176853.498706:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176853.498707:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176853.498709:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301749267
00000100:00000001:1.0:1786176853.498710:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301753037
00000100:00000001:1.0:1786176853.498711:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301749267
00000100:00000001:1.0:1786176853.498713:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301753037
00000100:00000001:1.0:1786176853.498714:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176853.498716:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.498717:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176853.498720:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176853/real 1786176853]  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176853.498725:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176853.498725:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176853.498726:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176853.498728:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176853.498729:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176853.498730:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176853.498732:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176853.498737:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498741:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176853.498742:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498743:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176853.498744:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176853.498746:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498750:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176853.498752:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498756:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176853.498757:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498758:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176853.498759:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498760:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176853.498761:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176853.498763:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176853.498777:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176853.498778:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176853.498781:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176853.498786:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176853.498788:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498789:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176853.498790:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856306304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176878 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176853.498794:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176853.498795:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176853.498796:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176853.498797:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176853.498798:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176853.498799:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176853.498800:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176853.498801:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176853.498802:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176853.498803:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176853.498804:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:1.0:1786176853.498805:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176853.498806:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498807:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176853.498809:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176853.498809:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176853.498811:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176853.498812:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176854.072596:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176854.072602:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176854.072606:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176854.072608:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176854.072610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176854.072611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176854.072660:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176854.072665:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176854.072667:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176854.520618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176854.520625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176854.520626:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176854.520631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176854.520634:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176854.520637:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176854.520639:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176854.520643:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176854.520646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176855.096602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176855.096607:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176855.096611:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176855.096612:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176855.096614:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176855.096615:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176855.096669:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176855.096672:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176855.096675:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176855.544600:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176855.544609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176855.544611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176855.544616:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176855.544618:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176855.544622:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176855.544624:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176855.544626:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176855.544628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176856.120630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176856.120638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176856.120644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176856.120645:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176856.120649:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176856.120650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176856.120742:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176856.120753:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176856.120756:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176856.568643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176856.568650:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176856.568652:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176856.568658:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176856.568662:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176856.568665:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176856.568668:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176856.568672:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176856.568675:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176857.144609:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176857.144621:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176857.144626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176857.144627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176857.144630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176857.144631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176857.144690:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176857.144695:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176857.144699:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786176857.336670:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301753037, at 4301753088
00000100:00000040:0.0:1786176857.336683:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301753037, at 4301753088
00000100:00000010:1.0:1786176857.336688:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b10c0.
00000100:00000010:0.0:1786176857.336704:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1ae0.
00000100:00000001:2.0:1786176857.592624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176857.592633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176857.592636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176857.592642:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176857.592646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176857.592649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176857.592652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176857.592655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176857.592658:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176858.168637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176858.168643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176858.168647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176858.168648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176858.168650:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176858.168651:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176858.168713:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176858.168717:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176858.168720:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176858.616640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1786176858.616640:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786176858.616649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176858.616654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176858.616654:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:2.0:1786176858.616661:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176858.616662:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1786176858.616666:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1786176858.616669:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176858.616670:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176858.616676:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176858.616678:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786176858.616679:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176858.616684:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176858.616690:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7081
00000400:00000200:3.0:1786176858.616702:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176858.616711:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176858.616715:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176858.616719:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.616723:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176858.616726:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.616729:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176858.616733:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176858.616736:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176858.616739:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176858.616743:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176858.616746:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176858.616749:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176858.616753:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.616756:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176858.616758:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.616761:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176858.616765:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176858.616769:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176858.616772:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176858.616775:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176858.616779:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176858.616783:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.616785:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176858.616788:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176858.616799:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d500.
00000020:00000040:3.0:1786176858.616803:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176858.616808:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176858.616810:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.616813:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176858.616816:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176858.616821:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176858.616830:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:3.0:1786176858.616838:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176858.616848:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176858.616859:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176858.616865:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176858.616870:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7c36393] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176858.616898:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176858.616902:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176858.616911:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176858.616913:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176858.616915:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176858.616918:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176858.616921:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7081
00000400:00000200:3.0:1786176858.616925:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176858.616927:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176858.616929:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176858.616932:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.616934:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176858.616935:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176858.616938:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786176858.616938:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176858.616942:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176858.616943:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1786176858.616944:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786176858.616947:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176858.616948:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1786176858.616949:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786176858.616953:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176858.616955:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786176858.616959:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176858.616960:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176858.616961:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176858.616965:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.616968:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176858.616972:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786176858.616977:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176858.616980:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176858.616984:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176858.616985:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176858.616988:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176858.616988:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176858.616993:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.616994:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786176858.616995:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:3.0:1786176858.616998:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786176858.616999:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:3.0:1786176858.617002:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176858.617003:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.617005:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176858.617010:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ea00.
00000100:00100000:1.0:1786176858.617011:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856306432:192.168.202.110@tcp:250:kworker.0
00000020:00000040:3.0:1786176858.617013:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786176858.617018:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176858.617019:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786176858.617022:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176858.617022:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.617024:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786176858.617026:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176858.617027:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176858.617030:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176858.617031:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176858.617048:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:3.0:1786176858.617051:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176858.617056:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:1.0:1786176858.617060:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
00000100:00000001:3.0:1786176858.617064:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1786176858.617065:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.617069:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176858.617072:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053162887] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176858.617079:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176858.617082:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176858.617084:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00080000:3.0:1786176858.617086:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176858.617090:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:1.0:1786176858.617091:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000001:3.0:1786176858.617091:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176858.617094:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176858.617097:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:1.0:1786176858.617099:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856306432, portal 25
00000100:00080000:3.0:1786176858.617100:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7081
00000100:00000001:1.0:1786176858.617105:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176858.617108:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000400:00000200:3.0:1786176858.617108:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176858.617113:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176858.617116:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786176858.617117:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176858.617125:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176858.617126:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786176858.617129:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786176858.617133:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786176858.617135:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:1.0:1786176858.617137:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000001:3.0:1786176858.617138:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:1.0:1786176858.617141:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856306432, offset 0
00000020:00000001:3.0:1786176858.617142:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176858.617144:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786176858.617145:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176858.617147:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:1.0:1786176858.617150:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:3.0:1786176858.617151:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786176858.617154:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786176858.617157:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.617159:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176858.617160:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1786176858.617163:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.617165:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176858.617167:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:1.0:1786176858.617169:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:3.0:1786176858.617173:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176858.617175:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176858.617177:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786176858.617178:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1786176858.617180:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176858.617183:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.617184:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:1.0:1786176858.617186:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00100000:3.0:1786176858.617187:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176858.617190:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.617192:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176858.617194:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000010:3.0:1786176858.617196:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c000.
00000020:00000040:3.0:1786176858.617201:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176858.617203:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176858.617204:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.617205:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176858.617206:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176858.617207:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176858.617210:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000010:3.0:1786176858.617211:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:3.0:1786176858.617215:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176858.617219:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254565:254565:256:4294967295] 192.168.202.110@tcp LPNI seq info [254554:254554:8:4294967295]
00000100:00100000:3.0:1786176858.617221:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176858.617233:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176858.617233:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176858.617237:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176858.617240:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006fe8fdb7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176858.617249:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786176858.617259:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176858.617264:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786176858.617265:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176858.617269:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786176858.617270:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176858.617271:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176858.617273:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176858.617275:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7081
00000800:00000010:1.0:1786176858.617276:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000400:00000200:3.0:1786176858.617280:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176858.617283:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176858.617285:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:1.0:1786176858.617289:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1786176858.617290:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.617294:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1786176858.617295:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000100:00000040:3.0:1786176858.617297:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.617301:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176858.617303:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000200:1.1:1786176858.617304:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:3.0:1786176858.617305:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786176858.617309:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:1.1:1786176858.617310:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000020:00000040:3.0:1786176858.617311:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786176858.617317:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000100:1.0:1786176858.617320:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:3.0:1786176858.617321:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000010:1.0:1786176858.617325:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000100:00000040:3.0:1786176858.617327:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.617330:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176858.617331:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176858.617332:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1786176858.617335:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176858.617339:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:3.0:1786176858.617339:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786176858.617345:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1786176858.617346:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000100:00000001:3.0:1786176858.617348:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176858.617350:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
02000000:00000001:3.0:1786176858.617351:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176858.617354:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:3.0:1786176858.617355:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176858.617359:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.617362:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000200:1.0:1786176858.617363:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1786176858.617367:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176858.617372:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.617375:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.617380:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000010:3.0:1786176858.617380:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c700.
00000020:00000040:3.0:1786176858.617384:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000040:1.0:1786176858.617386:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176858.617387:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176858.617389:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.617393:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176858.617395:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176858.617397:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.617399:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.617401:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:3.0:1786176858.617402:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000001:3.0:1786176858.617405:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.617406:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176858.617411:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176858.617413:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176858.617418:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301758157
00000100:00000001:3.0:1786176858.617423:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176858.617424:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176858.617428:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e56458ae] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1786176858.617431:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b10c0.
00000100:00000001:3.0:1786176858.617437:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176858.617439:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176858.617440:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:3.0:1786176858.617443:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786176858.617445:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176858.617447:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176858.617448:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301758157
00000100:00000001:3.0:1786176858.617450:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000010:1.0:1786176858.617452:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b13c0.
00000100:00080000:3.0:1786176858.617454:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7081
00000100:00000001:1.0:1786176858.617456:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000400:00000200:3.0:1786176858.617460:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176858.617462:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786176858.617465:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176858.617466:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176858.617468:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176858.617470:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:3.0:1786176858.617473:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.617475:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:1.0:1786176858.617477:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176858/real 1786176858]  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176858.617479:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.617481:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176858.617484:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176858.617486:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786176858.617489:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176858.617493:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000400:00000001:1.0:1786176858.617496:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:3.0:1786176858.617496:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786176858.617498:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000001:1.0:1786176858.617500:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:3.0:1786176858.617503:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:1.0:1786176858.617504:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000100:00000001:3.0:1786176858.617505:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176858.617527:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176858.617529:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176858.617530:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786176858.617533:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176858.617534:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176858.617536:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176858.617538:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786176858.617539:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
02000000:00000001:3.0:1786176858.617539:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.617540:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000010:1.0:1786176858.617543:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000008:00100000:3.0:1786176858.617544:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786176858.617547:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000008:00000001:3.0:1786176858.617548:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.617551:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176858.617554:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1786176858.617556:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e300.
00000020:00000040:3.0:1786176858.617562:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176858.617566:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:1.0:1786176858.617569:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176858.617569:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176858.617574:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176858.617576:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176858.617578:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.617583:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000010:3.0:1786176858.617584:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:1.0:1786176858.617586:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176858.617588:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.617589:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176858.617592:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176858.617596:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:1.0:1786176858.617597:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176858.617624:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176858.617625:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176858.617627:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176858.617632:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176858.617634:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e3752a60] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.617654:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176858.617656:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.617657:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.617660:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176858.617662:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176858.617662:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7092)
00000100:00000001:1.0:1786176858.617667:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176858.617669:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176858.617671:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176858.617674:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176858.617676:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176858.617681:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176858.617687:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856306432:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176858.617690:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.617692:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176858.617694:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856306432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.617698:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176858.617700:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176858.617702:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176858.617704:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176858.617705:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176858.617707:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176858.617708:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176858.617710:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:1.0:1786176858.617713:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176858.617714:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176858.617715:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:1.0:1786176858.617718:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176858.617719:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.617725:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.617731:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176858.617732:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.617733:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176858.617734:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176858.617736:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176858.617738:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.617743:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176858.617744:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176858.617745:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176858.617747:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176858.617748:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176858.617750:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176858.617753:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176858.617754:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176858.617755:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176858.617757:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176858.617759:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176858.617760:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176858.617763:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176858.617765:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176858.617768:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176858.617770:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856306560, portal 10
00000100:00000001:1.0:1786176858.617771:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176858.617772:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:1.0:1786176858.617777:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.617781:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176858.617783:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176858.617784:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000200:1.0:1786176858.617786:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856306560, offset 0
00000400:00000200:1.0:1786176858.617788:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176858.617791:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176858.617793:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176858.617795:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176858.617797:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176858.617798:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176858.617800:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176858.617802:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254566:254566:256:4294967295] 192.168.202.110@tcp LPNI seq info [254555:254555:8:4294967295]
00000400:00000200:1.0:1786176858.617805:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176858.617809:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176858.617812:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176858.617814:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176858.617817:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176858.617819:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786176858.617822:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176858.617824:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1786176858.617827:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176858.617828:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176858.617831:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176858.617833:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176858.617835:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:1.0:1786176858.617837:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176858.617838:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176858.617840:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.617846:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176858.617848:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.617852:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.617853:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176858.617855:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176858.617858:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176858.617860:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176858.617861:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301754378
00000100:00000001:1.0:1786176858.617863:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301758157
00000100:00000001:1.0:1786176858.617865:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301754378
00000100:00000001:1.0:1786176858.617866:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301758157
00000100:00000001:1.0:1786176858.617868:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176858.617869:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.617870:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176858.617874:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176858/real 1786176858]  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176858.617883:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176858.617884:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176858.617886:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176858.617889:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176858.617890:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176858.617893:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176858.617897:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176858.617906:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.617914:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176858.617915:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.617917:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176858.617919:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176858.617923:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.617929:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176858.617932:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.617937:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176858.617939:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.617940:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176858.617941:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.617942:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176858.617943:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176858.617945:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176858.617947:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176858.617949:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176858.617954:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176858.617961:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176858.617966:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176858.617969:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.617984:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176858.617986:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176858.617988:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176858.617990:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176858.617992:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176858.617995:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176858.617997:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176858.617999:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176858.618000:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176858.618001:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176858.618004:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176858.618005:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176858.618008:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176858.618009:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176858.618011:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176858.618013:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856306688, portal 10
00000100:00000001:1.0:1786176858.618015:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176858.618016:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:1.0:1786176858.618020:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618024:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176858.618026:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176858.618027:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000200:1.0:1786176858.618029:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856306688, offset 0
00000400:00000200:1.0:1786176858.618032:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176858.618050:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176858.618052:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176858.618055:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176858.618056:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176858.618058:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176858.618060:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176858.618062:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254567:254567:256:4294967295] 192.168.202.110@tcp LPNI seq info [254556:254556:8:4294967295]
00000400:00000200:1.0:1786176858.618065:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176858.618069:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176858.618072:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176858.618075:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176858.618078:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176858.618080:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786176858.618083:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176858.618085:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1786176858.618087:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176858.618089:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176858.618091:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176858.618093:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176858.618095:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:1.0:1786176858.618096:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176858.618098:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176858.618101:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618106:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176858.618108:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618112:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.618113:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176858.618116:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176858.618118:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176858.618120:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176858.618121:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301754503
00000100:00000001:1.0:1786176858.618123:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301758157
00000100:00000001:1.0:1786176858.618124:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301754503
00000100:00000001:1.0:1786176858.618126:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301758157
00000100:00000001:1.0:1786176858.618128:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176858.618129:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.618131:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176858.618133:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176858/real 1786176858]  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176858.618139:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176858.618139:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176858.618140:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176858.618142:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176858.618143:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176858.618144:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176858.618146:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176858.618152:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618155:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176858.618156:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618157:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176858.618158:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176858.618160:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618164:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176858.618167:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618171:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176858.618172:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618173:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176858.618174:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618176:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176858.618177:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176858.618179:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176858.618184:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176858.618186:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176858.618190:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176858.618199:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176858.618203:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176858.618206:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618213:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176858.618215:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176858.618217:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176858.618219:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176858.618221:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176858.618224:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176858.618228:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176858.618229:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176858.618230:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176858.618232:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176858.618234:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176858.618236:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176858.618238:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176858.618240:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176858.618243:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176858.618245:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856306816, portal 4
00000100:00000001:1.0:1786176858.618246:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176858.618247:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:1.0:1786176858.618251:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618256:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176858.618258:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176858.618259:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000200:1.0:1786176858.618260:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856306816, offset 0
00000400:00000200:1.0:1786176858.618263:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176858.618266:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176858.618268:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176858.618270:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176858.618272:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176858.618273:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176858.618275:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176858.618277:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254568:254568:256:4294967295] 192.168.202.110@tcp LPNI seq info [254557:254557:8:4294967295]
00000400:00000200:1.0:1786176858.618280:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176858.618284:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176858.618287:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176858.618289:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176858.618292:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176858.618294:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786176858.618296:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176858.618298:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1786176858.618301:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176858.618302:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176858.618304:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176858.618307:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176858.618309:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:1.0:1786176858.618310:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176858.618312:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176858.618314:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618319:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176858.618321:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618325:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.618326:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176858.618328:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176858.618331:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176858.618333:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176858.618334:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301754378
00000100:00000001:1.0:1786176858.618336:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301758157
00000100:00000001:1.0:1786176858.618337:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301754378
00000100:00000001:1.0:1786176858.618339:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301758157
00000100:00000001:1.0:1786176858.618340:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176858.618341:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.618342:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176858.618344:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176858/real 1786176858]  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176858.618350:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176858.618350:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176858.618351:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176858.618353:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176858.618354:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176858.618355:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176858.618357:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176858.618362:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618366:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176858.618367:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618368:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176858.618369:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176858.618371:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618375:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176858.618377:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618383:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176858.618384:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618385:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176858.618386:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618387:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176858.618388:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176858.618390:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176858.618391:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176858.618393:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176858.618395:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176858.618400:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176858.618402:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176858.618404:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618408:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176858.618408:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176858.618410:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176858.618411:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176858.618412:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176858.618414:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176858.618415:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176858.618417:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176858.618418:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176858.618419:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176858.618421:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176858.618422:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176858.618424:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176858.618426:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176858.618427:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176858.618429:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856306944, portal 4
00000100:00000001:1.0:1786176858.618430:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176858.618431:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:1.0:1786176858.618435:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618439:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176858.618440:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176858.618442:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000200:1.0:1786176858.618443:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856306944, offset 0
00000400:00000200:1.0:1786176858.618445:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176858.618447:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176858.618449:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176858.618451:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176858.618453:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176858.618454:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176858.618456:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176858.618458:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254569:254569:256:4294967295] 192.168.202.110@tcp LPNI seq info [254558:254558:8:4294967295]
00000400:00000200:1.0:1786176858.618460:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176858.618464:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176858.618467:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176858.618468:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176858.618470:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176858.618472:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786176858.618475:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176858.618476:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1786176858.618479:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176858.618480:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176858.618482:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176858.618484:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176858.618486:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:1.0:1786176858.618487:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176858.618489:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176858.618491:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618496:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176858.618498:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618502:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.618503:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176858.618505:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176858.618506:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176858.618508:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176858.618525:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301754387
00000100:00000001:1.0:1786176858.618527:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301758157
00000100:00000001:1.0:1786176858.618528:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301754387
00000100:00000001:1.0:1786176858.618529:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301758157
00000100:00000001:1.0:1786176858.618531:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176858.618532:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.618533:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176858.618536:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176858/real 1786176858]  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176858.618540:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176858.618541:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176858.618542:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176858.618544:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176858.618545:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176858.618546:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176858.618548:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176858.618554:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618562:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176858.618563:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618564:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176858.618565:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176858.618567:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618571:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176858.618573:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618577:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176858.618579:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618580:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176858.618580:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618582:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176858.618583:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176858.618585:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176858.618587:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176858.618588:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176858.618590:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176858.618595:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856306944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176858.618598:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618599:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176858.618601:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856306560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618605:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176858.618606:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176858.618608:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176858.618610:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176858.618611:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176858.618612:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176858.618613:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176858.618614:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:1.0:1786176858.618616:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176858.618617:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176858.618618:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:1.0:1786176858.618620:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176858.618621:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618622:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176858.618624:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856306688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618628:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176858.618629:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176858.618630:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176858.618631:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176858.618631:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176858.618632:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176858.618633:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176858.618634:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176858.618636:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176858.618636:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176858.618637:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:1.0:1786176858.618638:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176858.618639:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618640:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176858.618642:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856306816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618645:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176858.618646:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176858.618647:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176858.618648:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176858.618649:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176858.618650:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176858.618652:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176858.618652:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786176858.618654:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176858.618655:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176858.618656:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:1.0:1786176858.618657:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176858.618658:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618659:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176858.618660:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856306944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176858.618664:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176858.618665:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176858.618665:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176858.618668:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176858.618669:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176858.618669:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176858.618670:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176858.618671:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176858.618672:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176858.618673:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176858.618674:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:1.0:1786176858.618675:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176858.618676:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618677:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176858.618679:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176858.618680:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176858.618681:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176858.618682:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176859.192595:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176859.192602:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176859.192608:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176859.192609:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176859.192613:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176859.192614:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176859.192676:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176859.192679:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176859.192682:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176859.640615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176859.640621:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176859.640622:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176859.640627:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176859.640630:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176859.640633:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176859.640635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176859.640637:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176859.640639:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176860.216634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176860.216640:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176860.216642:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176860.216645:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176860.216649:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176860.216651:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176860.216653:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176860.216655:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176860.216657:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176860.216659:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176860.216659:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176860.216661:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176860.216662:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176860.216663:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176860.216664:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176860.216665:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176860.216667:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176860.216668:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176860.216734:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176860.216737:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176860.216740:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176860.664618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176860.664624:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176860.664626:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176860.664631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176860.664633:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176860.664636:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176860.664638:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176860.664640:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176860.664642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176861.240632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176861.240639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176861.240644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176861.240646:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176861.240649:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176861.240650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176861.240709:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176861.240714:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176861.240717:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176861.688610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176861.688617:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176861.688619:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176861.688623:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176861.688626:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176861.688629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176861.688632:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176861.688634:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176861.688636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176862.264633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176862.264639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176862.264643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176862.264644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176862.264646:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176862.264647:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176862.264700:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176862.264703:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176862.264705:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176862.456617:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301758157, at 4301758208
00000100:00000040:1.0:1786176862.456617:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301758157, at 4301758208
00000100:00000010:1.0:1786176862.456632:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b13c0.
00000100:00000010:0.0:1786176862.456635:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b10c0.
00000100:00000001:2.0:1786176862.712619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176862.712625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176862.712627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176862.712632:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176862.712635:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176862.712638:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176862.712640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176862.712642:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176862.712645:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176863.288608:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176863.288614:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176863.288618:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176863.288620:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176863.288622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176863.288625:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176863.288626:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176863.288629:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176863.288631:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176863.288633:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176863.288635:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176863.288637:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176863.288686:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176863.288689:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176863.288692:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786176863.736612:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786176863.736621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176863.736626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:3.0:1786176863.736627:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176863.736632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176863.736632:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176863.736636:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1786176863.736639:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176863.736641:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786176863.736646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176863.736650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176863.736652:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7087
00000100:00000001:2.0:1786176863.736656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176863.736660:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1786176863.736662:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176863.736666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176863.736671:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176863.736674:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176863.736677:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.736679:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176863.736681:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.736683:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176863.736686:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176863.736688:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176863.736690:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176863.736692:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176863.736694:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176863.736696:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176863.736699:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.736701:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176863.736703:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.736704:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176863.736707:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176863.736709:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176863.736710:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176863.736713:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176863.736715:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176863.736717:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176863.736718:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176863.736720:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176863.736728:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d180.
00000020:00000040:3.0:1786176863.736731:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176863.736734:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176863.736735:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176863.736738:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176863.736740:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176863.736744:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176863.736750:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786176863.736755:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176863.736763:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176863.736771:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176863.736775:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176863.736778:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085df314e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176863.736800:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176863.736803:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176863.736805:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176863.736806:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176863.736808:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176863.736810:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176863.736812:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7087
00000400:00000200:3.0:1786176863.736814:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176863.736816:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176863.736818:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176863.736820:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.736821:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176863.736823:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.736825:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176863.736826:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176863.736827:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786176863.736828:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176863.736830:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786176863.736832:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176863.736833:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786176863.736836:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.736837:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176863.736839:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.736841:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176863.736843:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1786176863.736843:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786176863.736849:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1786176863.736849:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176863.736854:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1786176863.736854:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176863.736858:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176863.736863:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1786176863.736864:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176863.736867:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176863.736872:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176863.736877:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1786176863.736878:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176863.736881:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1786176863.736882:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176863.736885:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1786176863.736885:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176863.736889:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:3.0:1786176863.736890:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e300.
02000000:00000001:1.0:1786176863.736892:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786176863.736893:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00100000:1.0:1786176863.736898:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856307072:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1786176863.736899:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176863.736903:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.736904:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1786176863.736908:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786176863.736909:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176863.736913:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176863.736913:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176863.736919:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176863.736924:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
02000000:00000001:1.0:1786176863.736925:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176863.736932:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
00000100:00000001:3.0:1786176863.736932:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176863.736937:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176863.736944:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1786176863.736944:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786176863.736949:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176863.736957:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000001:3.0:1786176863.736959:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176863.736962:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176863.736964:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856307072, portal 25
00000100:00000040:3.0:1786176863.736967:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e3752a60] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.736968:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176863.736971:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297088 : -114547945254528 : ffff97d1ba69d180)
00000100:00000040:1.0:1786176863.736981:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176863.736982:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737003:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1786176863.737004:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176863.737009:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000040:1.0:1786176863.737010:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176863.737013:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:1.0:1786176863.737015:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00080000:3.0:1786176863.737016:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:1.0:1786176863.737022:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856307072, offset 0
00000100:00000001:3.0:1786176863.737022:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176863.737027:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7087
00000400:00000200:1.0:1786176863.737033:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176863.737034:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176863.737043:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176863.737046:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176863.737049:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176863.737052:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176863.737054:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176863.737060:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176863.737061:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1786176863.737068:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176863.737069:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:3.0:1786176863.737073:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:1.0:1786176863.737076:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:3.0:1786176863.737078:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000200:1.0:1786176863.737081:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:3.0:1786176863.737083:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786176863.737085:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000040:3.0:1786176863.737088:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:1.0:1786176863.737091:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254570:254570:256:4294967295] 192.168.202.110@tcp LPNI seq info [254559:254559:8:4294967295]
00000020:00000001:3.0:1786176863.737095:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786176863.737096:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1786176863.737101:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786176863.737106:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.737112:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176863.737114:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176863.737116:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786176863.737116:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176863.737122:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786176863.737126:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:1.0:1786176863.737131:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176863.737131:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176863.737136:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:1.0:1786176863.737139:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
02000000:00000001:3.0:1786176863.737140:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176863.737143:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176863.737144:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176863.737147:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000800:00000200:1.0:1786176863.737148:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000008:00000001:3.0:1786176863.737152:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176863.737155:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:3.0:1786176863.737156:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176863.737163:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c700.
00000800:00000200:1.1:1786176863.737164:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:3.0:1786176863.737168:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000800:00000200:1.1:1786176863.737170:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
02000000:00000001:3.0:1786176863.737174:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176863.737177:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1786176863.737178:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176863.737182:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:1.0:1786176863.737183:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
02000000:00000001:3.0:1786176863.737184:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176863.737187:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176863.737190:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000010:3.0:1786176863.737191:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000400:00000200:1.0:1786176863.737197:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1786176863.737197:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176863.737206:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000100:00100000:3.0:1786176863.737207:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786176863.737209:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176863.737216:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1786176863.737217:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176863.737219:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176863.737222:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e56458ae] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176863.737228:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176863.737234:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176863.737240:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176863.737243:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1786176863.737244:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786176863.737247:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:1.0:1786176863.737249:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176863.737250:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176863.737257:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176863.737262:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7087
00000100:00000001:1.0:1786176863.737263:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737268:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000200:3.0:1786176863.737270:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176863.737274:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176863.737276:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176863.737276:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176863.737279:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.737283:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176863.737284:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1786176863.737288:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176863.737289:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301763277
00000100:00000001:3.0:1786176863.737291:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176863.737295:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176863.737296:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786176863.737297:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000010:1.0:1786176863.737298:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b13c0.
00000020:00000040:3.0:1786176863.737299:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786176863.737304:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786176863.737306:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176863.737309:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:3.0:1786176863.737309:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.737314:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176863.737316:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176863.737317:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301763277
00000100:00000001:3.0:1786176863.737318:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000010:1.0:1786176863.737323:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1ba0.
00000020:00000040:3.0:1786176863.737324:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786176863.737330:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176863.737331:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786176863.737335:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737337:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176863.737338:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176863.737342:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176863.737342:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176863.737347:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737348:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1786176863.737351:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176863.737355:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00100000:1.0:1786176863.737356:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176863/real 1786176863]  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:3.0:1786176863.737360:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176863.737364:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176863.737371:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c000.
00000400:00000001:1.0:1786176863.737372:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:3.0:1786176863.737374:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000400:00000001:1.0:1786176863.737375:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:3.0:1786176863.737378:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:1.0:1786176863.737379:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
02000000:00000001:3.0:1786176863.737381:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176863.737384:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000100:00000001:3.0:1786176863.737384:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176863.737386:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:1.0:1786176863.737388:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
02000000:00000001:3.0:1786176863.737389:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737394:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000010:3.0:1786176863.737395:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000200:1.0:1786176863.737399:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176863.737400:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176863.737416:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176863.737418:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176863.737428:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176863.737431:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.737436:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176863.737439:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176863.737440:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00100000:1.0:1786176863.737443:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176863.737446:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737454:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176863.737457:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006fe8fdb7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786176863.737461:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176863.737484:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737485:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176863.737488:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176863.737489:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.737495:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:3.0:1786176863.737495:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1786176863.737498:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176863.737499:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786176863.737502:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00080000:3.0:1786176863.737502:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176863.737505:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176863.737507:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786176863.737533:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786176863.737538:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7087
00000100:00080000:1.0:1786176863.737540:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176863.737545:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000400:00000200:3.0:1786176863.737546:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176863.737553:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:1.0:1786176863.737558:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176863.737558:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176863.737565:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.737568:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176863.737573:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.737578:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176863.737582:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176863.737584:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00100000:1.0:1786176863.737585:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856307072:192.168.202.110@tcp:250:kworker.0
00000020:00000001:3.0:1786176863.737586:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176863.737589:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786176863.737594:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786176863.737596:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176863.737598:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176863.737601:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176863.737603:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786176863.737604:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176863.737609:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176863.737613:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:1.0:1786176863.737614:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d180 x1872934856307072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786176863.737615:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786176863.737622:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176863.737627:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176863.737632:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176863.737635:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786176863.737636:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176863.737639:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000001:3.0:1786176863.737640:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176863.737644:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
00000100:00000001:3.0:1786176863.737644:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176863.737646:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000001:1.0:1786176863.737649:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000008:00000001:3.0:1786176863.737650:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1786176863.737653:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000001:3.0:1786176863.737653:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1786176863.737657:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000010:3.0:1786176863.737659:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ea00.
00000020:00000001:1.0:1786176863.737660:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176863.737666:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
00000020:00000040:3.0:1786176863.737666:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176863.737670:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786176863.737673:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786176863.737673:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176863.737676:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000001:3.0:1786176863.737677:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:1.0:1786176863.737679:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d180.
02000000:00000001:3.0:1786176863.737680:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176863.737683:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
02000000:00000001:3.0:1786176863.737684:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737686:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:3.0:1786176863.737689:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:1.0:1786176863.737690:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176863.737696:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737700:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00100000:3.0:1786176863.737702:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176863.737703:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737707:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176863.737710:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176863.737713:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176863.737720:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176863.737726:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176863.737730:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737732:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176863.737735:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176863.737737:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053162887] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786176863.737738:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176863.737742:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176863.737748:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176863.737751:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176863.737757:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856307200:192.168.202.110@tcp:38:kworker.0
00000100:00000040:3.0:1786176863.737758:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7097)
00000100:00000001:1.0:1786176863.737765:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176863.737768:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176863.737769:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176863.737772:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176863.737776:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176863.737779:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176863.737783:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176863.737786:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176863.737790:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176863.737793:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856307200, portal 10
00000100:00000001:1.0:1786176863.737795:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176863.737796:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:1.0:1786176863.737803:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.737811:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176863.737813:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176863.737816:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000200:1.0:1786176863.737818:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856307200, offset 0
00000400:00000200:1.0:1786176863.737822:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176863.737827:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176863.737831:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176863.737834:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176863.737837:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176863.737839:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176863.737842:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176863.737845:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254571:254571:256:4294967295] 192.168.202.110@tcp LPNI seq info [254560:254560:8:4294967295]
00000400:00000200:1.0:1786176863.737849:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176863.737856:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176863.737861:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176863.737865:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176863.737869:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176863.737872:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786176863.737877:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176863.737880:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1786176863.737884:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176863.737886:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176863.737889:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176863.737893:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176863.737896:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:1.0:1786176863.737898:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176863.737900:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176863.737904:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.737911:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176863.737913:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.737918:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737919:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176863.737921:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176863.737924:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176863.737926:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176863.737928:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301759498
00000100:00000001:1.0:1786176863.737930:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301763277
00000100:00000001:1.0:1786176863.737932:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301759498
00000100:00000001:1.0:1786176863.737933:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301763277
00000100:00000001:1.0:1786176863.737935:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176863.737937:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.737938:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176863.737940:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176863/real 1786176863]  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176863.737946:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176863.737948:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176863.737949:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176863.737950:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176863.737951:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176863.737953:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176863.737955:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176863.737961:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.737965:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176863.737965:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.737967:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176863.737967:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176863.737970:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.737974:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176863.737976:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.737982:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176863.737998:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.737999:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176863.738000:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738002:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176863.738003:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176863.738006:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176863.738008:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176863.738010:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176863.738013:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176863.738018:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856307200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176863.738021:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176863.738023:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738027:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176863.738028:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176863.738030:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176863.738031:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176863.738033:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176863.738035:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856307328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176863.738037:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176863.738038:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176863.738039:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176863.738040:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176863.738042:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176863.738044:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176863.738046:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176863.738048:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176863.738050:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176863.738052:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856307328, portal 10
00000100:00000001:1.0:1786176863.738053:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176863.738054:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:1.0:1786176863.738058:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738067:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176863.738068:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176863.738069:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000200:1.0:1786176863.738071:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856307328, offset 0
00000400:00000200:1.0:1786176863.738073:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176863.738076:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176863.738079:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176863.738081:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176863.738083:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176863.738085:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176863.738086:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176863.738088:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254572:254572:256:4294967295] 192.168.202.110@tcp LPNI seq info [254561:254561:8:4294967295]
00000400:00000200:1.0:1786176863.738091:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176863.738095:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176863.738098:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176863.738100:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176863.738103:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176863.738105:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786176863.738108:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176863.738109:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1786176863.738112:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176863.738113:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176863.738115:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176863.738118:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176863.738120:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:1.0:1786176863.738121:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176863.738122:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176863.738125:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738130:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176863.738132:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738141:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.738142:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176863.738144:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176863.738146:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176863.738148:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176863.738149:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301759623
00000100:00000001:1.0:1786176863.738151:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301763277
00000100:00000001:1.0:1786176863.738152:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301759623
00000100:00000001:1.0:1786176863.738154:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301763277
00000100:00000001:1.0:1786176863.738155:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176863.738156:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.738157:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176863.738159:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176863/real 1786176863]  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176863.738164:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176863.738165:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176863.738166:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176863.738167:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176863.738169:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176863.738170:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176863.738172:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176863.738177:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738181:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176863.738182:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738183:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176863.738184:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176863.738186:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738191:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176863.738194:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738198:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176863.738200:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738201:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176863.738201:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738202:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176863.738204:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176863.738205:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176863.738207:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176863.738208:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176863.738210:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176863.738215:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856307328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176863.738217:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176863.738219:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738223:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176863.738224:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176863.738226:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176863.738227:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176863.738228:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176863.738230:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856307456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176863.738232:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176863.738233:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176863.738234:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176863.738236:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176863.738237:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176863.738239:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176863.738241:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176863.738243:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176863.738245:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176863.738247:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856307456, portal 4
00000100:00000001:1.0:1786176863.738248:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176863.738249:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:1.0:1786176863.738253:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738257:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176863.738259:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176863.738260:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000200:1.0:1786176863.738261:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856307456, offset 0
00000400:00000200:1.0:1786176863.738264:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176863.738266:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176863.738268:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176863.738269:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176863.738271:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176863.738272:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176863.738274:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176863.738276:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254573:254573:256:4294967295] 192.168.202.110@tcp LPNI seq info [254562:254562:8:4294967295]
00000400:00000200:1.0:1786176863.738278:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176863.738282:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176863.738285:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176863.738286:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176863.738289:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176863.738290:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786176863.738293:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176863.738294:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1786176863.738297:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176863.738299:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176863.738300:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176863.738303:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176863.738304:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:1.0:1786176863.738305:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176863.738307:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176863.738309:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738314:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176863.738316:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738320:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.738321:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176863.738323:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176863.738325:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176863.738327:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176863.738328:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301759498
00000100:00000001:1.0:1786176863.738330:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301763277
00000100:00000001:1.0:1786176863.738331:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301759498
00000100:00000001:1.0:1786176863.738333:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301763277
00000100:00000001:1.0:1786176863.738334:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176863.738335:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.738336:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176863.738338:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176863/real 1786176863]  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176863.738343:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176863.738343:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176863.738344:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176863.738345:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176863.738346:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176863.738348:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176863.738350:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176863.738356:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738359:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176863.738360:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738361:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176863.738362:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176863.738364:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738368:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176863.738370:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738374:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176863.738375:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738376:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176863.738377:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738378:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176863.738379:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176863.738381:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176863.738382:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176863.738384:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176863.738386:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176863.738390:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856307456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176863.738392:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738394:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176863.738396:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856307200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738401:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176863.738402:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176863.738403:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176863.738405:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176863.738406:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176863.738407:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176863.738408:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176863.738409:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:1.0:1786176863.738411:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176863.738412:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176863.738413:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:1.0:1786176863.738414:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176863.738416:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738417:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176863.738419:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856307328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738423:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176863.738424:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176863.738425:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176863.738426:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176863.738426:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176863.738427:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176863.738428:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176863.738429:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176863.738430:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176863.738431:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176863.738432:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:1.0:1786176863.738433:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176863.738434:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738435:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176863.738437:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856307456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738441:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176863.738443:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176863.738444:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176863.738445:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176863.738446:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176863.738447:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176863.738448:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176863.738449:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786176863.738451:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176863.738451:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176863.738452:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:1.0:1786176863.738453:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176863.738454:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738455:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738457:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176863.738458:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.738459:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176863.738460:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176863.738461:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176863.738463:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738467:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176863.738468:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176863.738469:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176863.738471:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176863.738472:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176863.738474:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856307584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176863.738476:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176863.738477:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176863.738478:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176863.738479:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176863.738481:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176863.738482:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176863.738484:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176863.738486:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176863.738488:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176863.738489:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856307584, portal 4
00000100:00000001:1.0:1786176863.738491:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176863.738492:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:1.0:1786176863.738496:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738500:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176863.738502:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176863.738503:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
00000100:00000200:1.0:1786176863.738504:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856307584, offset 0
00000400:00000200:1.0:1786176863.738507:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176863.738524:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176863.738527:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176863.738529:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176863.738531:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176863.738533:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176863.738534:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176863.738536:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254574:254574:256:4294967295] 192.168.202.110@tcp LPNI seq info [254563:254563:8:4294967295]
00000400:00000200:1.0:1786176863.738539:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176863.738543:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176863.738547:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176863.738550:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176863.738554:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176863.738557:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786176863.738561:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176863.738563:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1786176863.738567:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176863.738569:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176863.738572:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176863.738576:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176863.738579:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:1.0:1786176863.738581:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:1.0:1786176863.738584:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176863.738588:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738603:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176863.738606:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738611:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.738612:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176863.738615:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176863.738617:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176863.738620:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176863.738621:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301759507
00000100:00000001:1.0:1786176863.738623:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301763277
00000100:00000001:1.0:1786176863.738624:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301759507
00000100:00000001:1.0:1786176863.738626:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301763277
00000100:00000001:1.0:1786176863.738628:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176863.738629:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.738630:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176863.738633:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176863/real 1786176863]  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176863.738638:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176863.738639:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176863.738640:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176863.738642:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176863.738643:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176863.738644:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176863.738646:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176863.738651:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738655:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176863.738656:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738658:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176863.738659:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176863.738661:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738665:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176863.738668:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738672:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176863.738673:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738674:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176863.738675:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738676:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176863.738677:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176863.738679:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176863.738681:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176863.738682:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176863.738684:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176863.738689:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856307584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176863.738691:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738692:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176863.738694:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856307584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176863.738698:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176863.738699:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176863.738701:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176863.738702:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176863.738703:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176863.738704:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176863.738706:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176863.738707:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176863.738709:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176863.738710:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176863.738711:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:1.0:1786176863.738713:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176863.738714:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738715:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176863.738718:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176863.738719:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176863.738720:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176863.738721:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176864.312630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176864.312635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176864.312639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176864.312640:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176864.312643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176864.312643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176864.312727:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176864.312733:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176864.312738:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176864.760642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176864.760649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176864.760650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176864.760656:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176864.760659:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176864.760662:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176864.760665:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176864.760667:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176864.760669:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176865.336602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176865.336610:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176865.336616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176865.336618:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176865.336621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176865.336623:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176865.336681:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176865.336686:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176865.336689:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176865.784597:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176865.784603:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176865.784605:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176865.784609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176865.784612:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176865.784615:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176865.784618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176865.784620:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176865.784622:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176866.360630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176866.360635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176866.360640:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176866.360641:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176866.360644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176866.360645:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176866.360690:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176866.360693:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176866.360696:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176866.808580:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176866.808586:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176866.808588:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176866.808592:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176866.808595:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176866.808598:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176866.808600:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176866.808603:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176866.808604:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176867.384605:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176867.384611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176867.384615:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176867.384616:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176867.384619:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176867.384620:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176867.384677:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176867.384681:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176867.384684:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176867.576617:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301763277, at 4301763328
00000100:00000040:1.0:1786176867.576617:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301763277, at 4301763328
00000100:00000010:1.0:1786176867.576632:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1ba0.
00000100:00000010:0.0:1786176867.576635:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b13c0.
00000100:00000001:2.0:1786176867.832602:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176867.832610:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176867.832612:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176867.832618:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176867.832622:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176867.832626:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176867.832629:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176867.832632:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176867.832635:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176868.408630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176868.408635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176868.408640:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176868.408641:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176868.408643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176868.408644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176868.408711:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176868.408715:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176868.408719:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176868.856619:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786176868.856623:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176868.856628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:2.0:1786176868.856630:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786176868.856631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.856636:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1786176868.856637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176868.856641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1786176868.856642:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176868.856644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176868.856651:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786176868.856651:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176868.856657:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176868.856661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176868.856662:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7092
00000400:00000200:2.0:1786176868.856674:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176868.856682:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176868.856686:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176868.856690:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.856695:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176868.856699:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.856702:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176868.856706:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176868.856709:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176868.856712:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176868.856715:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176868.856718:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176868.856721:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176868.856725:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.856727:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176868.856728:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.856729:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176868.856732:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176868.856734:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176868.856736:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176868.856738:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176868.856741:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176868.856743:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.856744:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176868.856746:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176868.856762:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000020:00000040:2.0:1786176868.856766:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176868.856769:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176868.856770:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.856772:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176868.856775:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176868.856778:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176868.856785:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786176868.856790:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176868.856798:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176868.856806:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176868.856809:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176868.856813:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176868.856833:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176868.856836:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176868.856838:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176868.856840:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176868.856841:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176868.856843:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176868.856845:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7092
00000400:00000200:2.0:1786176868.856848:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176868.856850:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176868.856852:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176868.856854:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.856856:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176868.856857:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.856859:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176868.856861:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176868.856862:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176868.856864:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176868.856865:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176868.856867:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176868.856868:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176868.856870:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.856872:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176868.856873:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.856874:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176868.856876:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786176868.856877:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:2.0:1786176868.856878:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176868.856882:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.856883:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1786176868.856884:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176868.856888:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:2.0:1786176868.856891:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176868.856894:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.856896:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:1.0:1786176868.856900:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1786176868.856900:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176868.856905:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.856910:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176868.856915:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000020:00000040:2.0:1786176868.856919:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786176868.856921:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1786176868.856923:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176868.856925:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176868.856925:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.856930:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176868.856932:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176868.856945:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176868.856946:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:2.0:1786176868.856951:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
02000000:00000001:1.0:1786176868.856952:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786176868.856956:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176868.856958:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176868.856963:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:1.0:1786176868.856967:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856307712:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786176868.856974:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176868.856977:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1786176868.856978:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176868.856984:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:2.0:1786176868.856984:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786176868.856989:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176868.856994:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786176868.856996:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176868.856999:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000010:1.0:1786176868.857001:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
00000100:00080000:2.0:1786176868.857002:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
02000000:00000001:1.0:1786176868.857006:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.857006:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176868.857010:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176868.857012:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786176868.857016:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786176868.857016:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7092
00000400:00000200:2.0:1786176868.857021:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176868.857022:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:2.0:1786176868.857026:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176868.857028:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176868.857031:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:1.0:1786176868.857033:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000001:2.0:1786176868.857033:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176868.857038:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.857040:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176868.857042:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176868.857043:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000100:00000200:1.0:1786176868.857044:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856307712, portal 25
00000020:00000001:2.0:1786176868.857044:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176868.857047:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000001:1.0:1786176868.857049:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176868.857052:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000020:00000001:2.0:1786176868.857060:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176868.857063:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:1.0:1786176868.857064:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176868.857067:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.857071:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176868.857074:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.857076:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176868.857078:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176868.857080:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176868.857081:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176868.857082:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176868.857084:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176868.857085:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.857087:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176868.857088:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176868.857094:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.857096:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786176868.857097:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176868.857102:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000100:00000040:1.0:1786176868.857104:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:2.0:1786176868.857106:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000010:1.0:1786176868.857110:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cc38.
02000000:00000001:2.0:1786176868.857110:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176868.857113:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.857115:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:1.0:1786176868.857116:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856307712, offset 0
02000000:00000001:2.0:1786176868.857117:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176868.857119:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176868.857123:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000400:00000200:1.0:1786176868.857127:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176868.857128:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176868.857134:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176868.857137:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176868.857146:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176868.857147:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1786176868.857149:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176868.857155:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1786176868.857155:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176868.857161:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176868.857167:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1786176868.857167:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176868.857171:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786176868.857172:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:2.0:1786176868.857177:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:1.0:1786176868.857179:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254575:254575:256:4294967295] 192.168.202.110@tcp LPNI seq info [254564:254564:8:4294967295]
00000100:00000001:2.0:1786176868.857180:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176868.857183:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786176868.857188:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176868.857189:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176868.857193:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7092
00000400:00000200:2.0:1786176868.857198:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176868.857203:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1786176868.857205:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176868.857210:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176868.857214:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:1.0:1786176868.857215:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176868.857216:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176868.857219:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000010:1.0:1786176868.857221:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000100:00000001:2.0:1786176868.857222:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176868.857226:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176868.857227:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176868.857228:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176868.857230:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000800:00000200:1.0:1786176868.857234:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786176868.857234:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176868.857240:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000200:1.1:1786176868.857244:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1786176868.857245:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.857251:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176868.857254:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.857257:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176868.857260:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:1.0:1786176868.857262:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176868.857263:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1786176868.857269:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176868.857270:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.857272:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301768397
02000000:00000001:2.0:1786176868.857274:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:1.0:1786176868.857280:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1ba0.
02000000:00000001:2.0:1786176868.857280:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176868.857286:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.857289:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786176868.857289:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176868.857293:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786176868.857296:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301768397
00000008:00000001:2.0:1786176868.857298:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786176868.857301:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1840.
00000100:00000001:2.0:1786176868.857303:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.857306:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000010:2.0:1786176868.857308:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000100:00000001:1.0:1786176868.857310:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1786176868.857312:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176868.857314:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176868.857317:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176868.857318:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176868.857320:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.857322:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176868.857323:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176868.857326:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176868.857328:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.857331:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000010:2.0:1786176868.857332:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:1.0:1786176868.857333:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1786176868.857336:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.857337:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176868.857341:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176868.857344:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00100000:2.0:1786176868.857345:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1786176868.857351:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176868.857362:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176868.857365:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176868.857372:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176868.857376:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176868.857378:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176868.857383:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176868.857386:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176868.857391:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176868.857396:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856307840:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786176868.857396:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.857402:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:2.0:1786176868.857402:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:1.0:1786176868.857406:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:2.0:1786176868.857407:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
02000000:00000001:1.0:1786176868.857408:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.857410:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:1.0:1786176868.857412:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:2.0:1786176868.857413:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
02000000:00000010:1.0:1786176868.857416:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
00000100:00000001:2.0:1786176868.857417:0:62275:0:(import.c:485:import_select_connection()) Process entered
02000000:00000001:1.0:1786176868.857421:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176868.857422:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7092
00000400:00000200:1.0:1786176868.857427:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176868.857429:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176868.857432:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d1b8.
00000400:00000200:2.0:1786176868.857433:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176868.857438:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:1.0:1786176868.857439:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8ca18.
00000100:00000040:2.0:1786176868.857440:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176868.857445:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1786176868.857446:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856307840, portal 10
00000100:00000040:2.0:1786176868.857448:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176868.857449:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176868.857452:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178944 : -114548255372672 : ffff97d1a7edca80)
00000100:00000001:2.0:1786176868.857452:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176868.857458:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176868.857460:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176868.857461:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000040:1.0:1786176868.857464:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176868.857465:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176868.857469:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176868.857474:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176868.857481:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176868.857482:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786176868.857484:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176868.857489:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176868.857491:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176868.857492:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176868.857496:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000400:00000010:1.0:1786176868.857498:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c2a8.
00000100:00080000:2.0:1786176868.857498:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176868.857504:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176868.857505:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856307840, offset 0
02000000:00000001:2.0:1786176868.857506:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786176868.857544:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176868.857549:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786176868.857552:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786176868.857555:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:2.0:1786176868.857557:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.857562:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:1.0:1786176868.857563:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00100000:2.0:1786176868.857566:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:1.0:1786176868.857569:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00000001:2.0:1786176868.857571:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176868.857573:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1786176868.857577:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176868.857578:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176868.857585:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254576:254576:255:4294967295] 192.168.202.110@tcp LPNI seq info [254565:254565:7:4294967295]
00000100:00000010:2.0:1786176868.857586:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786176868.857591:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000400:00000200:1.0:1786176868.857596:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786176868.857598:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176868.857602:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.857607:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176868.857611:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1786176868.857612:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1786176868.857614:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176868.857621:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000800:00000200:1.0:1786176868.857622:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176868.857627:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786176868.857629:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a600.
00000100:00100000:2.0:1786176868.857636:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.0:1786176868.857638:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176868.857649:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1786176868.857656:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176868.857657:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176868.857662:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176868.857664:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000100:00000040:2.0:1786176868.857670:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176868.857673:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176868.857679:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301764618
00000100:00000001:1.0:1786176868.857686:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301768397
00000100:00000001:2.0:1786176868.857688:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.857691:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301764618
00000100:00000001:1.0:1786176868.857695:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301768397
00000100:00000040:2.0:1786176868.857696:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7102)
00000100:00000001:1.0:1786176868.857698:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176868.857703:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.857714:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176868.857721:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176868.857728:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176868.857730:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176868.857733:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176868.857735:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176868.857738:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176868.857742:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856307968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176868.857746:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176868.857748:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176868.857749:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176868.857751:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176868.857756:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176868.857758:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176868.857762:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176868.857764:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786176868.857768:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176868.857771:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856307968, portal 10
00000100:00000001:1.0:1786176868.857774:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176868.857775:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000100:00000040:1.0:1786176868.857781:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176868.857788:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176868.857791:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176868.857793:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c198.
00000100:00000200:1.0:1786176868.857795:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856307968, offset 0
00000400:00000200:1.0:1786176868.857800:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176868.857805:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176868.857809:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176868.857812:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176868.857815:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176868.857817:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176868.857819:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176868.857822:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254577:254577:254:4294967295] 192.168.202.110@tcp LPNI seq info [254566:254566:6:4294967295]
00000400:00000200:1.0:1786176868.857826:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176868.857833:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176868.857837:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176868.857841:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ae00.
00000800:00000200:1.0:1786176868.857844:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176868.857848:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1786176868.857850:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176868.857853:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176868.857856:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176868.857858:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301764743
00000100:00000001:1.0:1786176868.857862:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301768397
00000100:00000001:1.0:1786176868.857864:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301764743
00000100:00000001:1.0:1786176868.857866:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301768397
00000100:00000001:1.0:1786176868.857868:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176868.857870:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.857873:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.857875:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176868.857878:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176868.857881:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1786176868.857883:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176868.857885:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176868.857886:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176868.857890:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176868.857897:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176868.857899:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176868.857901:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176868.857903:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176868.857905:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176868.857908:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856308096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176868.857911:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176868.857913:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176868.857915:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176868.857917:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176868.857921:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176868.857923:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176868.857926:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176868.857929:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176868.857952:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c4c8.
00000100:00000200:1.0:1786176868.857957:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856308096, portal 4
00000100:00000001:1.0:1786176868.857959:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176868.857961:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454188800 : -114548255362816 : ffff97d1a7edf100)
00000100:00000040:1.0:1786176868.857968:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176868.857975:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176868.857978:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176868.857981:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c7f8.
00000100:00000200:1.0:1786176868.857983:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856308096, offset 0
00000400:00000200:1.0:1786176868.857988:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176868.857992:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176868.857995:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176868.857998:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176868.858000:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176868.858002:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176868.858004:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176868.858007:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254578:254578:253:4294967295] 192.168.202.110@tcp LPNI seq info [254567:254567:5:4294967295]
00000400:00000200:1.0:1786176868.858011:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176868.858017:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176868.858022:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176868.858025:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881af00.
00000800:00000200:1.0:1786176868.858030:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176868.858034:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1786176868.858037:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176868.858041:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176868.858044:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176868.858046:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301764618
00000100:00000001:1.0:1786176868.858048:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301768397
00000100:00000001:1.0:1786176868.858051:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301764618
00000100:00000001:1.0:1786176868.858053:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301768397
00000100:00000001:1.0:1786176868.858055:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176868.858057:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.858059:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176868.858063:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176868.858075:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176868.858077:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176868.858079:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176868.858080:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176868.858082:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176868.858085:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856308224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176868.858087:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176868.858090:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176868.858091:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176868.858093:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176868.858096:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176868.858098:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176868.858101:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176868.858103:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8db58.
00000400:00000010:1.0:1786176868.858107:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c5d8.
00000100:00000200:1.0:1786176868.858110:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856308224, portal 4
00000100:00000001:1.0:1786176868.858112:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176868.858114:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000040:1.0:1786176868.858119:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176868.858125:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176868.858127:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176868.858129:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c088.
00000100:00000200:1.0:1786176868.858131:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856308224, offset 0
00000400:00000200:1.0:1786176868.858136:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176868.858140:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176868.858144:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176868.858147:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176868.858149:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176868.858151:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176868.858154:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176868.858157:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254579:254579:252:4294967295] 192.168.202.110@tcp LPNI seq info [254568:254568:4:4294967295]
00000400:00000200:1.0:1786176868.858161:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176868.858167:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176868.858171:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176868.858175:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ab00.
00000800:00000200:1.0:1786176868.858178:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176868.858182:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1786176868.858185:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176868.858187:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176868.858190:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176868.858192:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301764626
00000100:00000001:1.0:1786176868.858194:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301768397
00000100:00000001:1.0:1786176868.858196:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301764626
00000100:00000001:1.0:1786176868.858199:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301768397
00000100:00000001:1.0:1786176868.858201:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176868.858203:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.858207:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.858209:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176868.858212:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176868.858214:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1786176868.858216:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176868.858217:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176868.858219:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176868.858221:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176869.432620:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176869.432626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176869.432630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176869.432631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176869.432634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176869.432635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176869.432688:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176869.432691:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176869.432694:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176869.880678:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176869.880683:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176869.880685:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176869.880689:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176869.880692:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176869.880695:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176869.880698:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176869.880699:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176869.880701:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176870.456626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176870.456635:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176870.456638:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176870.456641:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176870.456646:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176870.456649:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176870.456651:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176870.456654:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176870.456665:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176870.456667:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176870.456669:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176870.456671:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176870.456673:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176870.456674:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176870.456676:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176870.456678:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176870.456682:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176870.456683:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176870.456753:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176870.456756:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176870.456759:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176870.904653:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176870.904669:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176870.904672:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176870.904677:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176870.904681:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176870.904685:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176870.904687:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176870.904689:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176870.904692:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176871.480621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176871.480628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176871.480635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176871.480636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176871.480640:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176871.480642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176871.480705:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176871.480708:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176871.480711:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176871.928764:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176871.928769:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176871.928771:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176871.928775:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176871.928778:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176871.928781:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176871.928784:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176871.928786:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176871.928788:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:2.0:1786176871.928897:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:2.0:1786176871.928972:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1786176871.928983:0:2382:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176871.928988:0:2382:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:2.0:1786176871.928993:0:2382:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176871.928998:0:2382:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1786176871.929003:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176871.929008:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:2.0:1786176871.929013:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176871.929017:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176871.929024:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cc38
00000400:00000010:2.0:1786176871.929026:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cc38.
00000100:00000001:2.0:1786176871.929032:0:2382:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176871.929040:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176871.929072:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176871.929077:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176871.929086:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176871.929088:0:2382:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176871.929098:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176871.929100:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a600.
00000400:00000200:2.0:1786176871.929103:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176871.929107:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176871.929111:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c2a8
00000400:00000010:2.0:1786176871.929113:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c2a8.
00000100:00000001:2.0:1786176871.929115:0:2382:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176871.929120:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176871.929128:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176871.929132:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176871.929139:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176871.929141:0:2382:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176871.929147:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176871.929150:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ae00.
00000400:00000200:2.0:1786176871.929153:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176871.929156:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176871.929159:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c198
00000400:00000010:2.0:1786176871.929161:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c198.
00000100:00000001:2.0:1786176871.929163:0:2382:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176871.929168:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176871.929183:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176871.929186:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176871.929192:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176871.929193:0:2382:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176871.929197:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176871.929200:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881af00.
00000400:00000200:2.0:1786176871.929202:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176871.929205:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176871.929208:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c7f8
00000400:00000010:2.0:1786176871.929209:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c7f8.
00000100:00000001:2.0:1786176871.929211:0:2382:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176871.929215:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176871.929222:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176871.929225:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176871.929231:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176871.929233:0:2382:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786176871.929237:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786176871.929240:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ab00.
00000400:00000200:2.0:1786176871.929243:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786176871.929246:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786176871.929248:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c088
00000400:00000010:2.0:1786176871.929251:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c088.
00000100:00000001:2.0:1786176871.929253:0:2382:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786176871.929258:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176871.929267:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176871.929270:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176871.929278:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176871.929280:0:2382:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176871.930000:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176871.930006:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176871.930009:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176871.930018:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176868/real 1786176871]  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176871.930032:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176871.930035:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176871.930038:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176871.930042:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c4c8
00000400:00000010:1.0:1786176871.930044:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c4c8.
00000100:00000001:1.0:1786176871.930048:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176871.930053:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176871.930064:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930072:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176871.930074:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930077:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176871.930080:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176871.930084:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930091:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176871.930097:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930105:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176871.930109:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930111:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176871.930113:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930117:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176871.930120:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176871.930124:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176871.930129:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176871.930133:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176871.930137:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176871.930146:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856308096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176871.930151:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176871.930154:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176868/real 1786176871]  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176871.930163:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176871.930164:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176871.930167:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8db58.
00000400:00000200:1.0:1786176871.930170:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c5d8
00000400:00000010:1.0:1786176871.930171:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c5d8.
00000100:00000001:1.0:1786176871.930174:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176871.930177:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176871.930185:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930192:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176871.930193:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930195:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176871.930196:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176871.930200:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930206:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176871.930211:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930218:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176871.930220:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930222:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176871.930223:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930225:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176871.930228:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176871.930231:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176871.930235:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176871.930238:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176871.930242:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176871.930250:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856308224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176871.930254:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176871.930258:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176868/real 1786176871]  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176871.930266:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176871.930268:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176871.930270:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d1b8.
00000400:00000200:1.0:1786176871.930273:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8ca18
00000400:00000010:1.0:1786176871.930275:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8ca18.
00000100:00000001:1.0:1786176871.930276:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176871.930279:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176871.930285:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930289:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176871.930290:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930291:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176871.930292:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176871.930294:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930301:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176871.930306:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930313:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176871.930315:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930317:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176871.930318:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930320:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176871.930322:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176871.930325:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176871.930329:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176871.930332:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176871.930336:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176871.930344:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856307840:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176871.930348:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176871.930352:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176868/real 1786176871]  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176871.930360:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176871.930362:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176871.930364:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786176871.930367:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176871.930368:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176871.930371:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176871.930374:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176871.930384:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930391:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176871.930392:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930394:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176871.930396:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176871.930399:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930406:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176871.930410:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930423:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176871.930426:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930428:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176871.930429:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930430:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176871.930433:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176871.930435:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176871.930440:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176871.930442:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176871.930446:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176871.930454:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856307968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176871.930459:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176871.930462:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176868/real 1786176871]  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176871.930469:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176871.930471:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176871.930473:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176871.930476:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176871.930477:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176871.930480:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176871.930482:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176871.930489:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930495:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176871.930496:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930498:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176871.930499:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176871.930502:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930508:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176871.930546:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930554:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176871.930556:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930558:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176871.930560:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930561:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176871.930563:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176871.930565:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176871.930569:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176871.930571:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176871.930575:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176871.930583:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856307712:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176871.930588:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930591:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176871.930594:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf100 x1872934856308096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930601:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176871.930604:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176871.930608:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176871.930612:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176871.930614:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176871.930616:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176871.930619:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176871.930622:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:1.0:1786176871.930626:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176871.930628:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176871.930629:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf100.
00000100:00000001:1.0:1786176871.930634:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176871.930635:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930637:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176871.930641:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856308224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930648:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176871.930650:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176871.930651:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176871.930655:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176871.930656:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176871.930657:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176871.930659:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176871.930661:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:1.0:1786176871.930664:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176871.930665:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176871.930666:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:1.0:1786176871.930669:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176871.930670:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930671:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176871.930675:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edca80 x1872934856307840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930682:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176871.930683:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176871.930685:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176871.930687:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176871.930689:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176871.930690:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176871.930691:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176871.930693:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:1.0:1786176871.930695:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176871.930709:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176871.930711:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edca80.
00000100:00000001:1.0:1786176871.930713:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176871.930714:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930716:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176871.930720:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856307968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930726:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176871.930728:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176871.930729:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176871.930732:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176871.930733:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176871.930735:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176871.930737:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176871.930738:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:1.0:1786176871.930741:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176871.930742:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176871.930743:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
00000100:00000001:1.0:1786176871.930745:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176871.930747:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930749:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176871.930752:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856307712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176871.930759:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176871.930761:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176871.930763:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176871.930765:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176871.930766:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176871.930767:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176871.930769:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176871.930771:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:1.0:1786176871.930773:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176871.930775:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176871.930776:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:1.0:1786176871.930778:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176871.930779:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930783:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176871.930790:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176871.930792:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176871.930794:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176871.930796:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176872.504595:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176872.504603:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176872.504609:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176872.504611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176872.504615:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176872.504616:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176872.504676:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176872.504680:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176872.504683:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176872.696641:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301768397, at 4301768448
00000100:00000040:1.0:1786176872.696641:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301768397, at 4301768448
00000100:00000010:0.0:1786176872.696698:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1ba0.
00000100:00000010:1.0:1786176872.696698:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1840.
00000100:00000001:3.0:1786176872.952710:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176872.952717:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176872.952720:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176872.952724:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176872.952728:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176872.952732:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176872.952735:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176872.952738:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176872.952740:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176873.528614:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176873.528619:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176873.528623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176873.528624:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176873.528626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176873.528635:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176873.528637:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176873.528639:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176873.528642:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176873.528644:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176873.528646:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176873.528647:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176873.528699:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176873.528703:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176873.528705:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176873.976619:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176873.976628:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176873.976631:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176873.976634:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176873.976639:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176873.976648:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7097
00000100:00000001:3.0:1786176873.976658:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000400:00000200:2.0:1786176873.976659:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176873.976663:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176873.976666:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786176873.976670:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00100000:3.0:1786176873.976671:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176873.976676:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1786176873.976677:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786176873.976679:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786176873.976684:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176873.976684:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176873.976687:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176873.976689:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786176873.976691:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176873.976695:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.976699:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176873.976702:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176873.976704:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176873.976708:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176873.976711:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176873.976714:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176873.976717:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176873.976721:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.976723:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176873.976725:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.976727:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176873.976730:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176873.976733:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176873.976736:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176873.976738:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176873.976742:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176873.976746:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176873.976748:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176873.976751:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176873.976757:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176873.976761:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176873.976765:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176873.976767:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176873.976770:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176873.976773:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176873.976777:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176873.976784:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786176873.976791:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176873.976801:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176873.976817:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176873.976822:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176873.976827:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176873.976849:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176873.976852:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176873.976856:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176873.976857:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176873.976859:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176873.976862:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176873.976864:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7097
00000400:00000200:2.0:1786176873.976868:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176873.976871:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176873.976873:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176873.976876:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.976878:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176873.976881:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.976883:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176873.976901:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176873.976903:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176873.976904:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176873.976907:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176873.976909:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176873.976911:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1786176873.976911:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176873.976919:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1786176873.976919:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.976923:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176873.976924:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786176873.976927:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.976933:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:1.0:1786176873.976935:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176873.976938:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176873.976947:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176873.976952:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.976954:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176873.976957:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176873.976957:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786176873.976963:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786176873.976964:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176873.976966:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176873.976969:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176873.976972:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176873.976973:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176873.976980:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00100000:1.0:1786176873.976981:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856308352:192.168.202.110@tcp:250:kworker.0
00000008:00000001:2.0:1786176873.976986:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176873.976991:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.976996:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:2.0:1786176873.976998:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
02000000:00000001:1.0:1786176873.977002:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:2.0:1786176873.977003:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786176873.977008:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176873.977008:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176873.977011:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176873.977014:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786176873.977015:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176873.977018:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176873.977020:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176873.977022:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000010:2.0:1786176873.977026:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
02000000:00000001:1.0:1786176873.977027:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176873.977034:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176873.977039:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1786176873.977044:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786176873.977047:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00000001:2.0:1786176873.977057:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000010:1.0:1786176873.977058:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000001:2.0:1786176873.977060:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176873.977067:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176873.977069:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856308352, portal 25
00000100:00000001:1.0:1786176873.977075:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176873.977078:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000001:2.0:1786176873.977087:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176873.977091:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176873.977092:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176873.977101:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176873.977105:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176873.977108:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176873.977111:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786176873.977113:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:1.0:1786176873.977119:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176873.977119:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7097
00000400:00000010:1.0:1786176873.977125:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000400:00000200:2.0:1786176873.977127:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:1.0:1786176873.977132:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856308352, offset 0
00000400:00000200:2.0:1786176873.977133:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176873.977139:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176873.977143:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.977148:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176873.977151:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.977153:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176873.977155:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:2.0:1786176873.977157:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176873.977160:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176873.977162:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176873.977164:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176873.977167:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786176873.977172:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1786176873.977172:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176873.977179:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.977181:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176873.977184:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1786176873.977187:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.977191:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176873.977195:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:2.0:1786176873.977196:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176873.977201:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1786176873.977203:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1786176873.977206:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176873.977209:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786176873.977211:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:2.0:1786176873.977213:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786176873.977217:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:2.0:1786176873.977219:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176873.977223:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:1.0:1786176873.977225:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254580:254580:256:4294967295] 192.168.202.110@tcp LPNI seq info [254569:254569:8:4294967295]
00000008:00100000:2.0:1786176873.977227:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176873.977234:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176873.977237:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176873.977238:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176873.977244:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786176873.977249:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:1.0:1786176873.977254:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1786176873.977255:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176873.977258:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176873.977262:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176873.977264:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176873.977267:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176873.977268:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000010:2.0:1786176873.977273:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000800:00000010:1.0:1786176873.977278:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000100:00000001:2.0:1786176873.977279:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176873.977288:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.0:1786176873.977294:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176873.977303:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176873.977306:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176873.977309:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:2.0:1786176873.977312:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.1:1786176873.977322:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176873.977328:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:2.0:1786176873.977330:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176873.977336:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000100:1.0:1786176873.977339:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176873.977342:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176873.977344:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176873.977345:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000010:1.0:1786176873.977348:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000100:00000001:2.0:1786176873.977350:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176873.977356:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7097
00000400:00000200:1.0:1786176873.977359:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786176873.977361:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176873.977368:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1786176873.977371:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:2.0:1786176873.977372:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176873.977379:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.977382:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176873.977383:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000100:00000040:2.0:1786176873.977386:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:1.0:1786176873.977388:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:2.0:1786176873.977391:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176873.977395:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:2.0:1786176873.977396:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176873.977400:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176873.977402:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000200:1.0:1786176873.977405:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176873.977406:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176873.977411:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176873.977416:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176873.977424:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.977428:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176873.977430:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176873.977432:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.977436:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:1.0:1786176873.977439:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176873.977440:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176873.977447:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176873.977455:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176873.977459:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176873.977462:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176873.977463:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176873.977467:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1786176873.977468:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176873.977472:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176873.977475:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786176873.977476:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1786176873.977481:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176873.977484:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176873.977486:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:2.0:1786176873.977489:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000100:00000001:1.0:1786176873.977492:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301773517
00000020:00000040:2.0:1786176873.977494:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176873.977499:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000010:1.0:1786176873.977500:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1840.
02000000:00000001:2.0:1786176873.977503:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176873.977506:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176873.977508:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176873.977535:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176873.977540:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786176873.977545:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.977547:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00100000:2.0:1786176873.977552:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176873.977557:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301773518
00000100:00000010:1.0:1786176873.977565:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1480.
00000100:00000001:2.0:1786176873.977566:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176873.977571:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.977573:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:2.0:1786176873.977577:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.977585:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786176873.977590:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.977591:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176873.977596:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176873.977597:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00080000:2.0:1786176873.977603:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176873.977606:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:1.0:1786176873.977607:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176873/real 1786176873]  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176873.977609:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176873.977616:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176873.977622:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7097
00000400:00000200:2.0:1786176873.977628:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000001:1.0:1786176873.977629:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:2.0:1786176873.977632:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000001:1.0:1786176873.977634:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:2.0:1786176873.977637:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176873.977641:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.977643:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:1.0:1786176873.977644:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000100:00000040:2.0:1786176873.977647:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176873.977650:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000100:00000001:2.0:1786176873.977651:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000010:1.0:1786176873.977654:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000020:00000001:2.0:1786176873.977656:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176873.977659:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:1.0:1786176873.977661:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:2.0:1786176873.977662:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176873.977666:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000200:1.0:1786176873.977668:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786176873.977670:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176873.977676:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176873.977684:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.977688:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1786176873.977689:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176873.977693:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176873.977699:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176873.977703:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:1.0:1786176873.977706:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00080000:2.0:1786176873.977708:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176873.977710:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176873.977713:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.977715:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:2.0:1786176873.977717:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176873.977719:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786176873.977722:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176873.977727:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176873.977728:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176873.977732:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176873.977737:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786176873.977743:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:2.0:1786176873.977743:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176873.977748:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176873.977751:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786176873.977755:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786176873.977760:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176873.977767:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176873.977768:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:2.0:1786176873.977770:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.977773:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176873.977775:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786176873.977777:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:2.0:1786176873.977779:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176873.977781:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786176873.977783:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.977786:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000010:2.0:1786176873.977789:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786176873.977793:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176873.977799:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176873.977807:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786176873.977811:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00080000:1.0:1786176873.977813:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:2.0:1786176873.977815:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786176873.977823:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000040:2.0:1786176873.977825:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.977829:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176873.977840:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176873.977841:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176873.977848:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7107)
00000100:00100000:1.0:1786176873.977858:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856308352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176873.977866:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.977869:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176873.977873:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856308352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.977881:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176873.977895:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176873.977898:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176873.977901:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176873.977903:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176873.977905:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176873.977908:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176873.977910:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:1.0:1786176873.977913:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176873.977915:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176873.977917:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:1.0:1786176873.977920:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176873.977921:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.977924:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.977932:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176873.977934:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.977936:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176873.977938:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176873.977939:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176873.977944:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.977952:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176873.977954:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176873.977956:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176873.977959:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176873.977961:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176873.977964:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856308480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176873.977968:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176873.977971:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176873.977972:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176873.977975:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176873.977978:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176873.977980:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176873.977985:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176873.977987:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176873.977991:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176873.977995:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856308480, portal 10
00000100:00000001:1.0:1786176873.977997:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176873.977999:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:1.0:1786176873.978007:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978015:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176873.978017:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176873.978020:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176873.978022:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856308480, offset 0
00000400:00000200:1.0:1786176873.978027:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176873.978032:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176873.978036:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176873.978039:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176873.978042:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176873.978054:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176873.978057:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176873.978060:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254581:254581:256:4294967295] 192.168.202.110@tcp LPNI seq info [254570:254570:8:4294967295]
00000400:00000200:1.0:1786176873.978065:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176873.978072:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176873.978077:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176873.978081:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176873.978085:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176873.978094:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786176873.978098:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176873.978102:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1786176873.978106:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176873.978108:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176873.978113:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176873.978117:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176873.978120:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176873.978122:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176873.978130:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176873.978134:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978143:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176873.978146:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978153:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.978154:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176873.978158:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176873.978163:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176873.978166:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176873.978168:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301769738
00000100:00000001:1.0:1786176873.978171:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301773517
00000100:00000001:1.0:1786176873.978173:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301769738
00000100:00000001:1.0:1786176873.978175:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301773518
00000100:00000001:1.0:1786176873.978177:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176873.978179:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.978182:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176873.978185:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176873/real 1786176873]  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176873.978193:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176873.978195:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176873.978197:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176873.978199:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176873.978201:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176873.978203:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176873.978207:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176873.978215:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978221:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176873.978223:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.978225:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176873.978227:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176873.978230:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978238:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176873.978242:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978249:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176873.978252:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.978253:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176873.978255:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.978256:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176873.978258:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176873.978261:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176873.978264:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176873.978266:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176873.978270:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176873.978277:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856308480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176873.978281:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176873.978284:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978295:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176873.978297:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176873.978299:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176873.978301:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176873.978302:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176873.978305:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856308608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176873.978308:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176873.978310:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176873.978312:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176873.978314:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176873.978317:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176873.978319:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176873.978322:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176873.978326:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176873.978329:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176873.978331:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856308608, portal 10
00000100:00000001:1.0:1786176873.978333:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176873.978334:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:1.0:1786176873.978339:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978346:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176873.978348:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176873.978350:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176873.978353:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856308608, offset 0
00000400:00000200:1.0:1786176873.978357:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176873.978361:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176873.978365:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176873.978368:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176873.978371:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176873.978373:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176873.978375:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176873.978379:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254582:254582:256:4294967295] 192.168.202.110@tcp LPNI seq info [254571:254571:8:4294967295]
00000400:00000200:1.0:1786176873.978384:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176873.978390:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176873.978394:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176873.978397:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176873.978401:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176873.978409:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786176873.978415:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176873.978418:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1786176873.978422:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176873.978424:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176873.978427:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176873.978431:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176873.978434:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176873.978436:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176873.978438:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176873.978442:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978449:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176873.978452:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978459:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.978460:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176873.978463:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176873.978466:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176873.978469:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176873.978472:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301769863
00000100:00000001:1.0:1786176873.978474:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301773517
00000100:00000001:1.0:1786176873.978477:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301769863
00000100:00000001:1.0:1786176873.978479:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301773518
00000100:00000001:1.0:1786176873.978481:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176873.978484:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.978486:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176873.978490:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176873/real 1786176873]  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176873.978499:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176873.978500:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176873.978502:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176873.978504:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176873.978506:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176873.978508:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176873.978528:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176873.978534:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978538:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176873.978539:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.978541:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176873.978542:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176873.978544:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978549:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176873.978551:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978557:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176873.978559:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.978561:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176873.978562:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.978563:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176873.978564:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176873.978566:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176873.978569:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176873.978571:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176873.978573:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176873.978578:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856308608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176873.978581:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176873.978583:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978587:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176873.978588:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176873.978590:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176873.978591:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176873.978592:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176873.978594:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856308736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176873.978596:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176873.978598:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176873.978599:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176873.978600:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176873.978603:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176873.978604:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176873.978607:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176873.978609:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176873.978611:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176873.978613:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856308736, portal 4
00000100:00000001:1.0:1786176873.978615:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176873.978616:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000040:1.0:1786176873.978620:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978624:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176873.978626:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176873.978628:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176873.978629:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856308736, offset 0
00000400:00000200:1.0:1786176873.978632:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176873.978635:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176873.978637:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176873.978639:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176873.978641:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176873.978643:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176873.978644:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176873.978651:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254583:254583:256:4294967295] 192.168.202.110@tcp LPNI seq info [254572:254572:8:4294967295]
00000400:00000200:1.0:1786176873.978654:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176873.978659:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176873.978662:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176873.978664:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176873.978667:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176873.978669:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786176873.978671:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176873.978673:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1786176873.978676:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176873.978677:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176873.978679:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176873.978682:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176873.978684:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176873.978685:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176873.978687:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176873.978689:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978694:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176873.978696:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978701:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.978702:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176873.978705:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176873.978707:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176873.978709:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176873.978710:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301769739
00000100:00000001:1.0:1786176873.978712:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301773517
00000100:00000001:1.0:1786176873.978714:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301769739
00000100:00000001:1.0:1786176873.978715:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301773518
00000100:00000001:1.0:1786176873.978716:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176873.978718:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.978719:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176873.978721:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176873/real 1786176873]  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176873.978725:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176873.978726:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176873.978727:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176873.978729:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176873.978730:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176873.978731:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176873.978733:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176873.978738:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978742:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176873.978743:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.978744:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176873.978745:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176873.978747:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978751:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176873.978753:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978758:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176873.978760:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.978761:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176873.978762:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.978763:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176873.978764:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176873.978765:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176873.978767:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176873.978768:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176873.978770:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176873.978775:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856308736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176873.978777:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176873.978779:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978783:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176873.978785:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176873.978787:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176873.978789:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176873.978791:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176873.978794:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856308864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176873.978797:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176873.978799:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176873.978800:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176873.978802:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176873.978805:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176873.978808:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176873.978811:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176873.978814:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176873.978818:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176873.978821:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856308864, portal 4
00000100:00000001:1.0:1786176873.978823:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176873.978825:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:1.0:1786176873.978832:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978839:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176873.978841:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176873.978843:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176873.978846:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856308864, offset 0
00000400:00000200:1.0:1786176873.978850:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176873.978854:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176873.978856:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176873.978858:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176873.978860:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176873.978862:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176873.978863:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176873.978866:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254584:254584:256:4294967295] 192.168.202.110@tcp LPNI seq info [254573:254573:8:4294967295]
00000400:00000200:1.0:1786176873.978868:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176873.978872:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176873.978875:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176873.978878:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176873.978881:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176873.978882:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786176873.978900:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176873.978902:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1786176873.978906:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176873.978908:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176873.978911:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176873.978915:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176873.978918:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176873.978920:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176873.978923:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176873.978928:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978936:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176873.978940:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176898 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.978947:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.978948:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176873.978952:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176873.978955:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176873.978957:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176873.978959:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301769747
00000100:00000001:1.0:1786176873.978961:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301773517
00000100:00000001:1.0:1786176873.978963:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301769747
00000100:00000001:1.0:1786176873.978965:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301773518
00000100:00000001:1.0:1786176873.978967:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176873.978969:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.978971:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176873.978975:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176873/real 1786176873]  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176873.978983:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176873.978984:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176873.978986:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176873.978989:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176873.978990:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176873.978992:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176873.978996:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176873.979005:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.979018:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176873.979020:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.979021:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176873.979022:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176873.979025:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.979029:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176873.979032:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.979037:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176873.979038:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.979039:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176873.979040:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.979041:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176873.979042:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176873.979044:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176873.979045:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176873.979047:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176873.979050:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176873.979058:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856308864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176873.979062:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.979064:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176873.979067:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856308480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.979074:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176873.979076:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176873.979079:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176873.979081:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176873.979083:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176873.979085:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176873.979087:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176873.979089:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:1.0:1786176873.979092:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176873.979093:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176873.979095:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:1.0:1786176873.979097:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176873.979099:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.979101:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176873.979104:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856308608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.979111:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176873.979112:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176873.979113:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176873.979115:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176873.979115:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176873.979117:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176873.979118:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176873.979119:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:1.0:1786176873.979120:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176873.979121:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176873.979122:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:1.0:1786176873.979123:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176873.979124:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.979125:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176873.979127:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856308736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.979131:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176873.979132:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176873.979133:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176873.979134:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176873.979135:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176873.979136:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176873.979138:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176873.979139:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:1.0:1786176873.979140:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176873.979141:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176873.979141:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:1.0:1786176873.979143:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176873.979144:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.979145:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176873.979146:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856308864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176898 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176873.979150:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176873.979151:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176873.979152:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176873.979154:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176873.979154:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176873.979155:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176873.979156:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176873.979157:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:1.0:1786176873.979158:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176873.979159:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176873.979160:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:1.0:1786176873.979161:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176873.979162:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.979163:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176873.979166:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176873.979167:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176873.979168:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176873.979170:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176874.552611:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176874.552617:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176874.552621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176874.552622:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176874.552624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176874.552625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176874.552689:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176874.552694:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176874.552698:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176875.000708:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176875.000714:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176875.000716:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176875.000721:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176875.000723:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176875.000726:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176875.000729:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176875.000731:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176875.000733:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176875.576638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176875.576643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176875.576647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176875.576648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176875.576651:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176875.576652:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176875.576710:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176875.576716:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176875.576719:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.0:1786176875.896661:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:3.0:1786176875.896668:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:3.0:1786176875.896673:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:0.0:1786176875.896841:0:2384:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1786176875.896847:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786176875.896859:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1786176875.896896:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786176875.896900:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176876.024663:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176876.024669:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176876.024671:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176876.024675:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176876.024678:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176876.024681:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176876.024683:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176876.024685:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176876.024687:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176876.600625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176876.600632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176876.600636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176876.600638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176876.600640:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176876.600641:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176876.600688:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176876.600692:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176876.600695:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176877.048583:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176877.048590:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176877.048592:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176877.048598:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176877.048601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176877.048605:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176877.048608:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176877.048612:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176877.048615:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176877.624609:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176877.624616:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176877.624622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176877.624624:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176877.624627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176877.624629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176877.624681:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176877.624724:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176877.624727:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176877.816700:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301773517, at 4301773568
00000100:00000040:1.0:1786176877.816700:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301773518, at 4301773568
00000100:00000010:0.0:1786176877.816728:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1840.
00000100:00000010:1.0:1786176877.816733:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1480.
00000100:00000001:3.0:1786176878.072604:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176878.072612:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176878.072615:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176878.072622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176878.072627:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176878.072632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176878.072636:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176878.072639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176878.072642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176878.648576:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176878.648583:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176878.648588:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176878.648589:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176878.648592:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176878.648593:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176878.648639:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176878.648643:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176878.648646:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786176879.096607:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176879.096615:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176879.096618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176879.096618:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176879.096621:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176879.096626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176879.096626:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786176879.096631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176879.096637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176879.096641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176879.096645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176879.096648:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1786176879.096652:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7102
00000100:00000001:2.0:1786176879.096653:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176879.096658:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176879.096662:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176879.096668:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176879.096671:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176879.096674:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.096677:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176879.096679:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.096682:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176879.096684:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176879.096686:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176879.096689:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176879.096691:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176879.096693:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176879.096695:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176879.096698:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.096699:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176879.096701:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.096702:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176879.096704:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176879.096706:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176879.096707:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176879.096710:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176879.096713:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176879.096715:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.096717:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176879.096718:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176879.096725:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d500.
00000020:00000040:3.0:1786176879.096728:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176879.096731:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176879.096732:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.096734:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176879.096737:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176879.096740:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176879.096747:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:3.0:1786176879.096752:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176879.096760:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176879.096768:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176879.096771:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176879.096774:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7c36393] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176879.096788:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176879.096790:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176879.096793:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176879.096794:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176879.096795:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176879.096797:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176879.096798:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7102
00000400:00000200:3.0:1786176879.096801:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176879.096803:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176879.096805:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176879.096808:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.096810:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176879.096812:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.096813:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176879.096815:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176879.096816:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1786176879.096816:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786176879.096818:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176879.096822:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:1.0:1786176879.096823:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1786176879.096824:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176879.096828:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176879.096829:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176879.096830:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.096833:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176879.096847:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.096850:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:1.0:1786176879.096851:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786176879.096852:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786176879.096857:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176879.096862:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176879.096865:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176879.096871:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1786176879.096872:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176879.096876:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176879.096876:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.096882:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786176879.096884:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:3.0:1786176879.096886:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786176879.096889:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:3.0:1786176879.096890:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176879.096896:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.096896:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176879.096899:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f100.
00000100:00100000:1.0:1786176879.096904:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856308992:192.168.202.110@tcp:250:kworker.0
00000020:00000040:3.0:1786176879.096905:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176879.096909:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176879.096912:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.096915:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176879.096921:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176879.096924:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176879.096927:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1786176879.096929:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176879.096932:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:1.0:1786176879.096934:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176879.096938:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176879.096940:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176879.096947:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:3.0:1786176879.096950:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000400:00000010:1.0:1786176879.096953:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00000001:3.0:1786176879.096955:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176879.096962:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00100000:3.0:1786176879.096963:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176879.096972:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1786176879.096974:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856308992, portal 25
00000100:00000001:3.0:1786176879.096975:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.096980:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000040:3.0:1786176879.096982:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1ef9f45] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.096984:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000100:00000001:3.0:1786176879.096995:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176879.096997:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176879.096998:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176879.097005:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786176879.097007:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176879.097010:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176879.097015:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176879.097020:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1786176879.097020:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7102
00000100:00000040:1.0:1786176879.097028:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176879.097028:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176879.097035:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000400:00000200:3.0:1786176879.097035:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176879.097039:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:1.0:1786176879.097042:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856308992, offset 0
00000100:00000040:3.0:1786176879.097043:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097047:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176879.097049:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097052:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176879.097055:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:1.0:1786176879.097057:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:3.0:1786176879.097057:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786176879.097061:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176879.097063:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786176879.097065:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786176879.097066:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786176879.097070:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176879.097072:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176879.097072:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176879.097077:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097079:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176879.097081:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:1.0:1786176879.097084:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:3.0:1786176879.097087:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097091:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176879.097093:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786176879.097094:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1786176879.097096:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176879.097099:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.097101:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:1.0:1786176879.097104:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00100000:3.0:1786176879.097104:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176879.097109:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176879.097113:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1786176879.097113:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176879.097121:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000010:3.0:1786176879.097121:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f480.
00000020:00000040:3.0:1786176879.097126:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176879.097129:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:1.0:1786176879.097132:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254585:254585:256:4294967295] 192.168.202.110@tcp LPNI seq info [254574:254574:8:4294967295]
02000000:00000001:3.0:1786176879.097132:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.097136:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176879.097138:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176879.097142:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176879.097146:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:3.0:1786176879.097148:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000400:00000200:1.0:1786176879.097158:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786176879.097159:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176879.097167:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.0:1786176879.097172:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097179:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000010:1.0:1786176879.097182:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000100:00000001:3.0:1786176879.097182:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176879.097190:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092aa1aa7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1786176879.097196:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786176879.097202:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176879.097204:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00080000:3.0:1786176879.097205:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176879.097212:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786176879.097213:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176879.097214:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176879.097219:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.1:1786176879.097219:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786176879.097225:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7102
00000800:00000200:1.1:1786176879.097227:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000400:00000200:3.0:1786176879.097231:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176879.097238:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000100:1.0:1786176879.097240:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786176879.097241:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176879.097244:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097245:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176879.097247:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097248:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176879.097250:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176879.097250:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786176879.097251:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176879.097253:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786176879.097254:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786176879.097255:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786176879.097257:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097258:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176879.097259:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097260:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176879.097261:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786176879.097263:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097264:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176879.097265:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176879.097267:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176879.097268:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.097269:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176879.097271:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176879.097272:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.097273:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176879.097276:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ca80.
00000020:00000040:3.0:1786176879.097277:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176879.097279:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176879.097280:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.097281:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176879.097282:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176879.097284:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176879.097288:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000800:00000010:1.0:1786176879.097294:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000100:00000001:3.0:1786176879.097294:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176879.097304:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176879.097305:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176879.097317:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1786176879.097323:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176879.097326:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000100:00000001:3.0:1786176879.097328:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176879.097330:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176879.097336:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:3.0:1786176879.097336:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ffb0865] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176879.097344:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176879.097347:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176879.097352:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176879.097358:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786176879.097361:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176879.097363:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176879.097367:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176879.097369:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1786176879.097370:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7102
00000400:00000200:3.0:1786176879.097377:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1786176879.097378:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786176879.097381:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176879.097386:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176879.097392:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097397:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176879.097402:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176879.097402:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097405:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176879.097408:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176879.097409:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786176879.097410:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176879.097411:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786176879.097413:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176879.097415:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:3.0:1786176879.097415:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786176879.097420:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176879.097421:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176879.097423:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176879.097424:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.097425:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176879.097430:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786176879.097433:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1786176879.097435:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176879.097436:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176879.097440:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176879.097442:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301778637
02000000:00000001:3.0:1786176879.097443:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176879.097448:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.097449:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176879.097451:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000010:1.0:1786176879.097452:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1480.
00000008:00000001:3.0:1786176879.097453:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.097458:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176879.097461:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69fb80.
00000100:00000001:1.0:1786176879.097463:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000040:3.0:1786176879.097464:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176879.097469:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176879.097470:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176879.097472:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176879.097473:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176879.097474:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176879.097477:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:1.0:1786176879.097479:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301778637
00000100:00000001:3.0:1786176879.097480:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176879.097488:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:1.0:1786176879.097489:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b18a0.
00000100:00000001:1.0:1786176879.097496:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786176879.097496:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176879.097501:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176879.097505:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014f55bbb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176879.097533:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176879.097538:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7112)
00000100:00000001:1.0:1786176879.097539:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176879.097545:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.097547:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176879.097554:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176879/real 1786176879]  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176879.097564:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176879.097566:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176879.097568:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176879.097572:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176879.097573:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176879.097576:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176879.097580:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176879.097589:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.097595:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176879.097597:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.097599:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176879.097601:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176879.097605:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.097612:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176879.097617:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.097624:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176879.097627:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.097629:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176879.097630:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.097633:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176879.097635:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176879.097638:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176879.097641:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176879.097644:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176879.097648:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176879.097657:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856308992:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176879.097662:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.097664:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176879.097667:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856308992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.097679:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176879.097681:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176879.097684:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176879.097689:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176879.097691:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176879.097693:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176879.097696:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176879.097698:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:1.0:1786176879.097701:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176879.097703:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176879.097705:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:1.0:1786176879.097708:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176879.097709:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.097711:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.097718:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176879.097719:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.097722:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176879.097723:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176879.097725:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176879.097728:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.097734:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176879.097736:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176879.097738:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176879.097740:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176879.097742:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176879.097745:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856309120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176879.097748:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176879.097751:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176879.097752:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176879.097754:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176879.097757:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176879.097759:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176879.097763:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176879.097767:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176879.097771:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176879.097774:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856309120, portal 10
00000100:00000001:1.0:1786176879.097776:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176879.097777:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:1.0:1786176879.097784:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.097791:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176879.097793:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176879.097795:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176879.097798:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856309120, offset 0
00000400:00000200:1.0:1786176879.097802:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176879.097806:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176879.097810:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176879.097813:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176879.097816:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176879.097818:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176879.097821:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176879.097824:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254586:254586:256:4294967295] 192.168.202.110@tcp LPNI seq info [254575:254575:8:4294967295]
00000400:00000200:1.0:1786176879.097828:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176879.097847:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176879.097853:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176879.097857:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176879.097861:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176879.097864:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786176879.097869:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176879.097872:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1786176879.097876:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176879.097877:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176879.097881:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176879.097884:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176879.097887:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176879.097889:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176879.097892:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176879.097896:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.097904:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176879.097907:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.097913:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.097915:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176879.097918:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176879.097923:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176879.097926:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176879.097928:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301774858
00000100:00000001:1.0:1786176879.097931:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301778637
00000100:00000001:1.0:1786176879.097934:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301774858
00000100:00000001:1.0:1786176879.097936:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301778637
00000100:00000001:1.0:1786176879.097938:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176879.097940:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.097942:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176879.097945:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176879/real 1786176879]  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176879.097953:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176879.097954:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176879.097956:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176879.097958:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176879.097960:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176879.097962:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176879.097965:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176879.097973:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.097986:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176879.097988:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.097989:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176879.097991:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176879.097995:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098002:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176879.098006:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098012:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176879.098015:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098016:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176879.098017:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098019:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176879.098021:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176879.098024:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176879.098026:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176879.098029:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176879.098032:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176879.098040:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856309120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176879.098044:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176879.098048:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098054:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176879.098056:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176879.098058:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176879.098060:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176879.098062:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176879.098065:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856309248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176879.098068:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176879.098070:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176879.098072:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176879.098074:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176879.098077:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176879.098079:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176879.098082:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176879.098085:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176879.098088:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176879.098091:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856309248, portal 10
00000100:00000001:1.0:1786176879.098093:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176879.098094:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:1.0:1786176879.098118:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098124:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176879.098127:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176879.098129:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176879.098132:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856309248, offset 0
00000400:00000200:1.0:1786176879.098135:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176879.098139:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176879.098143:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176879.098146:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176879.098149:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176879.098151:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176879.098154:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176879.098157:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254587:254587:256:4294967295] 192.168.202.110@tcp LPNI seq info [254576:254576:8:4294967295]
00000400:00000200:1.0:1786176879.098161:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176879.098168:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176879.098173:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176879.098176:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176879.098180:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176879.098183:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786176879.098188:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176879.098191:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1786176879.098195:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176879.098197:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176879.098200:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176879.098203:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176879.098206:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176879.098208:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176879.098211:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176879.098215:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098222:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176879.098226:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098233:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.098235:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176879.098238:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176879.098241:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176879.098244:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176879.098246:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301774983
00000100:00000001:1.0:1786176879.098248:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301778637
00000100:00000001:1.0:1786176879.098250:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301774983
00000100:00000001:1.0:1786176879.098252:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301778637
00000100:00000001:1.0:1786176879.098254:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176879.098256:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.098257:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176879.098261:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176879/real 1786176879]  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176879.098269:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176879.098270:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176879.098272:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176879.098275:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176879.098276:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176879.098278:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176879.098281:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176879.098290:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098296:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176879.098297:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098299:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176879.098301:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176879.098304:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098311:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176879.098315:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098321:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176879.098323:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098325:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176879.098327:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098329:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176879.098330:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176879.098333:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176879.098336:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176879.098338:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176879.098343:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176879.098350:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856309248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176879.098354:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176879.098358:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098364:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176879.098365:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176879.098368:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176879.098370:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176879.098372:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176879.098375:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856309376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176879.098378:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176879.098380:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176879.098381:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176879.098383:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176879.098386:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176879.098388:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176879.098391:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176879.098394:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176879.098397:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176879.098400:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856309376, portal 4
00000100:00000001:1.0:1786176879.098402:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176879.098403:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:1.0:1786176879.098409:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098415:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176879.098417:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176879.098419:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176879.098421:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856309376, offset 0
00000400:00000200:1.0:1786176879.098426:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176879.098429:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176879.098432:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176879.098435:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176879.098438:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176879.098440:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176879.098443:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176879.098446:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254588:254588:256:4294967295] 192.168.202.110@tcp LPNI seq info [254577:254577:8:4294967295]
00000400:00000200:1.0:1786176879.098450:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176879.098456:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176879.098460:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176879.098463:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176879.098467:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176879.098470:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786176879.098474:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176879.098476:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1786176879.098479:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176879.098481:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176879.098484:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176879.098487:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176879.098490:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176879.098492:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176879.098494:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176879.098497:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098505:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176879.098508:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098532:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.098534:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176879.098537:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176879.098541:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176879.098544:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176879.098546:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301774859
00000100:00000001:1.0:1786176879.098548:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301778637
00000100:00000001:1.0:1786176879.098551:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301774859
00000100:00000001:1.0:1786176879.098553:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301778637
00000100:00000001:1.0:1786176879.098555:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176879.098556:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.098558:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176879.098562:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176879/real 1786176879]  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176879.098569:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176879.098571:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176879.098572:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176879.098574:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176879.098576:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176879.098578:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176879.098582:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176879.098590:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098596:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176879.098597:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098599:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176879.098600:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176879.098604:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098611:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176879.098614:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098621:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176879.098624:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098625:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176879.098627:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098628:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176879.098630:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176879.098633:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176879.098635:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176879.098637:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176879.098642:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176879.098650:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856309376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176879.098654:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176879.098657:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098663:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176879.098665:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176879.098667:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176879.098669:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176879.098670:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176879.098673:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856309504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176879.098676:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176879.098677:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176879.098679:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176879.098681:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176879.098684:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176879.098685:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176879.098688:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176879.098691:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176879.098694:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176879.098697:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856309504, portal 4
00000100:00000001:1.0:1786176879.098699:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176879.098700:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000040:1.0:1786176879.098706:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098713:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176879.098715:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176879.098717:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176879.098719:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856309504, offset 0
00000400:00000200:1.0:1786176879.098723:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176879.098727:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176879.098730:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176879.098733:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176879.098736:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176879.098738:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176879.098740:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176879.098743:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254589:254589:256:4294967295] 192.168.202.110@tcp LPNI seq info [254578:254578:8:4294967295]
00000400:00000200:1.0:1786176879.098747:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176879.098754:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176879.098759:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176879.098762:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176879.098766:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176879.098769:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786176879.098774:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176879.098776:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1786176879.098780:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176879.098782:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176879.098786:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176879.098789:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176879.098792:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176879.098794:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176879.098796:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176879.098800:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098814:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176879.098817:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176904 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098823:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.098824:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176879.098827:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176879.098830:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176879.098833:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176879.098848:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301774867
00000100:00000001:1.0:1786176879.098852:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301778637
00000100:00000001:1.0:1786176879.098853:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301774867
00000100:00000001:1.0:1786176879.098856:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301778637
00000100:00000001:1.0:1786176879.098858:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176879.098860:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.098862:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176879.098865:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176879/real 1786176879]  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176879.098873:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176879.098875:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176879.098876:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176879.098879:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176879.098880:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176879.098883:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176879.098886:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176879.098894:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098901:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176879.098903:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098905:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176879.098906:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176879.098910:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098917:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176879.098920:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098927:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176879.098929:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098930:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176879.098932:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098934:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176879.098935:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176879.098938:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176879.098941:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176879.098944:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176879.098948:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176879.098955:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856309504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176879.098959:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098961:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176879.098964:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856309120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.098970:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176879.098972:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176879.098974:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176879.098977:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176879.098978:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176879.098980:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176879.098982:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176879.098984:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:1.0:1786176879.098987:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176879.098988:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176879.098989:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:1.0:1786176879.098992:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176879.098994:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.098995:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176879.098999:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856309248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.099005:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176879.099006:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176879.099007:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176879.099010:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176879.099011:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176879.099012:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176879.099014:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176879.099015:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176879.099018:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176879.099019:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176879.099020:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:1.0:1786176879.099022:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176879.099024:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.099025:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176879.099028:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856309376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.099034:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176879.099036:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176879.099037:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176879.099039:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176879.099041:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176879.099042:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176879.099044:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176879.099045:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786176879.099047:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176879.099049:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176879.099050:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:1.0:1786176879.099052:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176879.099054:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.099055:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176879.099058:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856309504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176904 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176879.099064:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176879.099065:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176879.099067:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176879.099069:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176879.099070:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176879.099072:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176879.099073:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176879.099075:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176879.099077:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176879.099078:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176879.099079:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:1.0:1786176879.099081:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176879.099083:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.099085:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176879.099088:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176879.099089:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176879.099091:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176879.099093:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176879.672607:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176879.672614:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176879.672615:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176879.672618:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176879.672622:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176879.672625:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176879.672626:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176879.672628:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176879.672631:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176879.672631:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176879.672632:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176879.672634:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176879.672635:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176879.672636:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176879.672638:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176879.672639:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176879.672642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176879.672643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176879.672723:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176879.672726:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176879.672729:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176880.120624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176880.120630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176880.120633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176880.120638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176880.120642:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176880.120646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176880.120649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176880.120652:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176880.120655:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176880.696675:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176880.696684:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176880.696690:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176880.696691:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176880.696693:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176880.696694:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176880.696789:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176880.696795:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176880.696800:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176881.144632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176881.144640:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176881.144643:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176881.144648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176881.144652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176881.144656:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176881.144660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176881.144662:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176881.144665:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176881.720684:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176881.720690:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176881.720695:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176881.720697:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176881.720701:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176881.720702:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176881.720754:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176881.720758:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176881.720761:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176882.168632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176882.168650:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176882.168653:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176882.168662:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176882.168666:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176882.168670:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176882.168673:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176882.168676:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176882.168679:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176882.744682:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176882.744694:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176882.744702:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176882.744703:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176882.744706:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176882.744709:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176882.744711:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176882.744715:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176882.744718:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176882.744721:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176882.744722:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176882.744724:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176882.744790:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176882.744794:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176882.744797:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176882.936592:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301778637, at 4301778688
00000100:00000040:1.0:1786176882.936592:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301778637, at 4301778688
00000100:00000010:1.0:1786176882.936611:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b18a0.
00000100:00000010:0.0:1786176882.936616:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1480.
00000100:00000001:2.0:1786176883.192662:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176883.192672:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176883.192674:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176883.192680:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176883.192683:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176883.192687:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176883.192690:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176883.192692:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176883.192695:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176883.768623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176883.768630:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176883.768636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176883.768637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176883.768640:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176883.768642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176883.768733:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176883.768740:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176883.768744:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786176884.216658:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786176884.216670:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1786176884.216670:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176884.216680:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176884.216680:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1786176884.216689:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176884.216691:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1786176884.216696:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176884.216701:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176884.216701:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786176884.216708:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176884.216713:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1786176884.216715:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7107
00000100:00000001:2.0:1786176884.216717:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176884.216724:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176884.216734:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176884.216744:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176884.216748:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176884.216753:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176884.216758:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176884.216762:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176884.216765:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176884.216769:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176884.216771:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176884.216774:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176884.216778:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176884.216780:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176884.216783:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176884.216818:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176884.216820:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176884.216822:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176884.216824:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176884.216828:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176884.216841:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176884.216844:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176884.216847:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176884.216851:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176884.216854:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176884.216856:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176884.216859:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176884.216869:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f800.
00000020:00000040:3.0:1786176884.216873:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176884.216877:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176884.216879:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176884.216882:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176884.216885:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176884.216889:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176884.216898:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786176884.216904:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176884.216914:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176884.216925:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176884.216930:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176884.216935:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4d339d9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176884.216962:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176884.216966:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176884.216974:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176884.216975:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176884.216977:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176884.216980:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176884.216984:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7107
00000400:00000200:3.0:1786176884.216989:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176884.216994:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176884.216996:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176884.217000:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176884.217004:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786176884.217004:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176884.217008:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176884.217010:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176884.217012:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176884.217015:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1786176884.217018:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176884.217021:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786176884.217024:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000040:1.0:1786176884.217026:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786176884.217027:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786176884.217032:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176884.217038:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176884.217047:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:3.0:1786176884.217048:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176884.217051:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176884.217054:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1786176884.217057:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1786176884.217059:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176884.217062:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176884.217064:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:1.0:1786176884.217068:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786176884.217072:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00100000:1.0:1786176884.217077:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856309632:192.168.202.110@tcp:250:kworker.0
00000100:00080000:3.0:1786176884.217082:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176884.217085:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1786176884.217089:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176884.217092:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176884.217094:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786176884.217096:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176884.217100:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176884.217104:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:3.0:1786176884.217104:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176884.217108:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176884.217113:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176884.217117:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176884.217119:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000010:1.0:1786176884.217122:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000008:00100000:3.0:1786176884.217126:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1786176884.217130:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000008:00000001:3.0:1786176884.217132:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176884.217137:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176884.217140:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856309632, portal 25
00000100:00000001:1.0:1786176884.217144:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000010:3.0:1786176884.217144:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d880.
00000100:00000001:1.0:1786176884.217148:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
00000020:00000040:3.0:1786176884.217151:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000040:1.0:1786176884.217159:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176884.217159:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176884.217163:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176884.217168:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176884.217172:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176884.217176:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.217177:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176884.217182:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000010:3.0:1786176884.217185:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000400:00000010:1.0:1786176884.217186:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176884.217190:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856309632, offset 0
00000100:00000001:3.0:1786176884.217194:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176884.217198:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176884.217208:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176884.217214:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:3.0:1786176884.217216:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176884.217219:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176884.217226:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176884.217232:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1786176884.217234:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176884.217237:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1786176884.217239:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176884.217247:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097f9b8db] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176884.217248:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254590:254590:256:4294967295] 192.168.202.110@tcp LPNI seq info [254579:254579:8:4294967295]
00000400:00000200:1.0:1786176884.217263:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176884.217270:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176884.217276:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786176884.217281:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1786176884.217283:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786176884.217287:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176884.217289:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:1.0:1786176884.217290:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176884.217296:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:1.0:1786176884.217298:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000100:00080000:3.0:1786176884.217302:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7107
00000800:00000200:1.0:1786176884.217307:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:3.0:1786176884.217312:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:1.0:1786176884.217313:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000400:00000200:3.0:1786176884.217321:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:1.1:1786176884.217323:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786176884.217327:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:1.1:1786176884.217328:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1786176884.217337:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786176884.217337:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000010:1.0:1786176884.217342:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000100:00000001:3.0:1786176884.217343:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176884.217348:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1786176884.217349:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176884.217353:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176884.217355:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:3.0:1786176884.217360:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:1.0:1786176884.217362:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000020:00000040:3.0:1786176884.217363:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000010:1.0:1786176884.217366:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000020:00000001:3.0:1786176884.217367:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176884.217372:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:3.0:1786176884.217372:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786176884.217377:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000200:1.0:1786176884.217378:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786176884.217383:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786176884.217391:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176884.217393:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786176884.217395:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786176884.217400:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176884.217403:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176884.217409:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176884.217411:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.217414:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:3.0:1786176884.217415:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000001:1.0:1786176884.217422:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176884.217422:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1786176884.217429:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176884.217429:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.217433:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301783757
02000000:00000001:3.0:1786176884.217435:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:1.0:1786176884.217439:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b18a0.
02000000:00000001:3.0:1786176884.217442:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176884.217446:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:3.0:1786176884.217447:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176884.217453:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176884.217455:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176884.217458:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176884.217459:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.217463:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301783757
00000100:00000010:3.0:1786176884.217465:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c380.
00000100:00000010:1.0:1786176884.217468:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1240.
00000020:00000040:3.0:1786176884.217470:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786176884.217472:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:3.0:1786176884.217477:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176884.217478:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176884.217481:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.217482:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.217488:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1786176884.217489:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176884.217492:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:1.0:1786176884.217494:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176884/real 1786176884]  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176884.217496:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176884.217506:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000010:3.0:1786176884.217506:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000400:00000001:1.0:1786176884.217508:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1786176884.217536:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176884.217541:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176884.217546:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000100:00100000:3.0:1786176884.217546:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786176884.217549:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176884.217553:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176884.217559:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176884.217567:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1786176884.217571:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176884.217572:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.217581:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176884.217584:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176884.217584:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000833bf1c9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.217586:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176884.217588:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176884.217596:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176884.217605:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.217613:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176884.217613:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176884.217621:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000040:1.0:1786176884.217623:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176884.217624:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176884.217630:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176884.217638:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176884.217639:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176884.217644:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176884.217646:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7107
00000100:00000001:1.0:1786176884.217648:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176884.217652:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.217655:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176884.217657:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000400:00000200:3.0:1786176884.217657:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1786176884.217664:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000400:00000200:3.0:1786176884.217668:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786176884.217671:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00080000:3.0:1786176884.217675:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176884.217676:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786176884.217684:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176884.217687:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176884.217689:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176884.217697:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176884.217705:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00100000:1.0:1786176884.217708:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856309632:192.168.202.110@tcp:250:kworker.0
00000020:00000001:3.0:1786176884.217710:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176884.217714:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:1.0:1786176884.217717:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:3.0:1786176884.217718:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176884.217720:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000040:3.0:1786176884.217724:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000040:1.0:1786176884.217725:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856309632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786176884.217734:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176884.217735:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176884.217738:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000020:00000001:3.0:1786176884.217740:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
02000000:00000010:1.0:1786176884.217741:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176884.217745:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176884.217748:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176884.217750:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000040:3.0:1786176884.217750:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000020:00000001:1.0:1786176884.217756:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00000001:3.0:1786176884.217758:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000010:1.0:1786176884.217759:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:1.0:1786176884.217764:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000040:3.0:1786176884.217765:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176884.217766:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176884.217769:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000001:3.0:1786176884.217771:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176884.217773:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176884.217775:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.217778:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:3.0:1786176884.217778:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786176884.217799:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176884.217802:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.217804:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:3.0:1786176884.217805:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176884.217807:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.217811:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:3.0:1786176884.217811:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176884.217812:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176884.217815:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:3.0:1786176884.217817:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176884.217821:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176884.217822:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176884.217827:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786176884.217832:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000008:00000001:3.0:1786176884.217834:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.217835:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176884.217840:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1786176884.217840:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176884.217842:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176884.217845:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176884.217849:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856309760:192.168.202.110@tcp:38:kworker.0
00000100:00000010:3.0:1786176884.217849:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ed80.
00000100:00000001:1.0:1786176884.217855:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176884.217857:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:3.0:1786176884.217857:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:1.0:1786176884.217859:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176884.217863:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1786176884.217866:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000010:1.0:1786176884.217867:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:3.0:1786176884.217870:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176884.217871:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176884.217878:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176884.217878:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:1.0:1786176884.217882:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
02000000:00000001:3.0:1786176884.217883:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:1.0:1786176884.217886:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
02000000:00000001:3.0:1786176884.217888:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176884.217890:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856309760, portal 10
00000100:00000001:1.0:1786176884.217894:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176884.217895:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
02000000:00000010:3.0:1786176884.217899:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000040:1.0:1786176884.217901:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176884.217908:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.217909:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176884.217913:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176884.217916:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176884.217919:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856309760, offset 0
00000100:00100000:3.0:1786176884.217922:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176884.217924:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176884.217930:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176884.217936:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176884.217941:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176884.217945:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1786176884.217945:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176884.217952:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1786176884.217952:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176884.217957:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176884.217962:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254591:254591:256:4294967295] 192.168.202.110@tcp LPNI seq info [254580:254580:8:4294967295]
00000100:00000040:3.0:1786176884.217963:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ccd6c6d2] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176884.217969:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176884.217982:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786176884.217993:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176884.217995:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786176884.217999:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:1.0:1786176884.218002:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000100:00080000:3.0:1786176884.218008:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:1.0:1786176884.218009:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786176884.218012:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:1.0:1786176884.218015:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00080000:3.0:1786176884.218016:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176884.218022:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.1:1786176884.218024:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786176884.218029:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7107
00000800:00000200:1.1:1786176884.218030:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000400:00000200:3.0:1786176884.218038:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000100:1.0:1786176884.218041:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176884.218045:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:3.0:1786176884.218046:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1786176884.218049:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:3.0:1786176884.218051:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786176884.218056:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176884.218061:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000100:00000040:3.0:1786176884.218061:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:1.0:1786176884.218065:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:3.0:1786176884.218067:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176884.218068:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:3.0:1786176884.218072:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:1.0:1786176884.218076:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176884.218077:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176884.218083:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176884.218086:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:1.0:1786176884.218089:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000001:3.0:1786176884.218090:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000040:1.0:1786176884.218092:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786176884.218095:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786176884.218102:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176884.218103:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.218106:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:3.0:1786176884.218109:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176884.218110:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176884.218116:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000100:00000040:3.0:1786176884.218117:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176884.218122:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176884.218123:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176884.218125:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301779978
00000100:00000040:3.0:1786176884.218129:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176884.218131:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301783757
00000100:00000001:3.0:1786176884.218133:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176884.218137:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301779978
00000020:00000040:3.0:1786176884.218139:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:1.0:1786176884.218144:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301783757
00000100:00080000:3.0:1786176884.218145:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176884.218151:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176884.218151:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.218158:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176884.218158:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176884.218163:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000001:3.0:1786176884.218164:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176884.218168:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176884.218171:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176884/real 1786176884]  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176884.218171:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176884.218175:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176884.218181:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176884.218186:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176884.218189:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176884.218192:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000010:3.0:1786176884.218193:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e680.
00000400:00000010:1.0:1786176884.218194:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176884.218199:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000020:00000040:3.0:1786176884.218199:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000400:00000010:1.0:1786176884.218202:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
02000000:00000001:3.0:1786176884.218206:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176884.218207:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:3.0:1786176884.218211:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176884.218212:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176884.218216:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176884.218219:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176884.218223:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176884.218225:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1786176884.218230:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:1.0:1786176884.218232:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176884.218236:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176884.218237:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.218240:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176884.218243:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786176884.218246:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:1.0:1786176884.218252:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176884.218265:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176884.218270:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.218272:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176884.218279:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1839131] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786176884.218282:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218312:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176884.218313:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.218320:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176884.218322:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7117)
00000100:00000001:1.0:1786176884.218323:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176884.218327:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.218330:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176884.218333:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176884.218336:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176884.218342:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176884.218345:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176884.218351:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176884.218361:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856309760:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176884.218365:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176884.218368:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218375:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176884.218377:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176884.218380:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176884.218382:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176884.218383:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176884.218386:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856309888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176884.218388:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176884.218389:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176884.218391:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176884.218392:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176884.218395:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176884.218396:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176884.218399:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176884.218401:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176884.218403:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176884.218406:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856309888, portal 10
00000100:00000001:1.0:1786176884.218407:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176884.218408:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764293504 : -114547945258112 : ffff97d1ba69c380)
00000100:00000040:1.0:1786176884.218412:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218417:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176884.218419:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176884.218420:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176884.218422:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856309888, offset 0
00000400:00000200:1.0:1786176884.218424:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176884.218427:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176884.218430:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176884.218432:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176884.218434:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176884.218436:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176884.218437:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176884.218440:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254592:254592:256:4294967295] 192.168.202.110@tcp LPNI seq info [254581:254581:8:4294967295]
00000400:00000200:1.0:1786176884.218442:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176884.218447:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176884.218450:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176884.218453:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176884.218455:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176884.218457:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786176884.218460:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176884.218462:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1786176884.218465:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176884.218466:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176884.218468:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176884.218471:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176884.218473:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176884.218474:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176884.218476:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176884.218478:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218484:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176884.218486:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218490:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.218491:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176884.218493:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176884.218495:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176884.218498:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176884.218499:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301780103
00000100:00000001:1.0:1786176884.218501:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301783757
00000100:00000001:1.0:1786176884.218502:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301780103
00000100:00000001:1.0:1786176884.218503:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301783757
00000100:00000001:1.0:1786176884.218505:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176884.218506:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.218507:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176884.218525:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176884/real 1786176884]  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176884.218530:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176884.218531:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176884.218532:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176884.218534:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176884.218535:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176884.218537:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176884.218539:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176884.218544:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218549:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176884.218549:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.218551:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176884.218552:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176884.218554:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218559:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176884.218561:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218566:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176884.218568:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.218569:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176884.218570:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.218571:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176884.218572:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176884.218574:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176884.218575:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176884.218577:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176884.218579:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176884.218584:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856309888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176884.218588:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.218590:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176884.218594:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856309760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218600:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176884.218602:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176884.218604:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176884.218606:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176884.218608:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176884.218610:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176884.218612:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176884.218613:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:1.0:1786176884.218616:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176884.218618:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176884.218619:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:1.0:1786176884.218622:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176884.218624:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.218626:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176884.218630:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c380 x1872934856309888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218637:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176884.218638:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176884.218640:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176884.218642:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176884.218643:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176884.218645:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176884.218647:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176884.218648:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176884.218651:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176884.218652:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176884.218653:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c380.
00000100:00000001:1.0:1786176884.218656:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176884.218657:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.218659:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.218663:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176884.218665:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.218666:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176884.218668:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176884.218669:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176884.218674:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218681:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176884.218683:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176884.218685:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176884.218687:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176884.218689:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176884.218691:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856310016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176884.218694:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176884.218695:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176884.218696:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176884.218697:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176884.218700:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176884.218701:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176884.218704:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176884.218706:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176884.218708:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176884.218710:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856310016, portal 4
00000100:00000001:1.0:1786176884.218712:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176884.218713:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:1.0:1786176884.218717:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218722:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176884.218723:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176884.218724:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176884.218726:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856310016, offset 0
00000400:00000200:1.0:1786176884.218728:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176884.218731:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176884.218733:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176884.218735:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176884.218737:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176884.218739:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176884.218740:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176884.218743:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254593:254593:256:4294967295] 192.168.202.110@tcp LPNI seq info [254582:254582:8:4294967295]
00000400:00000200:1.0:1786176884.218746:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176884.218750:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176884.218752:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176884.218754:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176884.218757:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176884.218759:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786176884.218761:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176884.218764:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1786176884.218767:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176884.218768:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176884.218770:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176884.218773:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176884.218775:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176884.218776:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176884.218778:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176884.218780:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218800:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176884.218804:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218809:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.218810:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176884.218813:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176884.218816:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176884.218818:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176884.218819:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301779979
00000100:00000001:1.0:1786176884.218821:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301783757
00000100:00000001:1.0:1786176884.218822:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301779979
00000100:00000001:1.0:1786176884.218824:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301783757
00000100:00000001:1.0:1786176884.218825:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176884.218826:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.218828:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176884.218830:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176884/real 1786176884]  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176884.218835:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176884.218837:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176884.218838:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176884.218841:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176884.218843:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176884.218845:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176884.218848:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176884.218863:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218870:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176884.218872:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.218874:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176884.218875:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176884.218880:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218887:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176884.218891:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218899:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176884.218901:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.218903:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176884.218904:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.218905:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176884.218907:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176884.218908:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176884.218911:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176884.218912:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176884.218915:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176884.218920:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856310016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176884.218923:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176884.218925:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218933:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176884.218934:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176884.218936:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176884.218937:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176884.218938:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176884.218940:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856310144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176884.218942:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176884.218943:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176884.218945:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176884.218946:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176884.218948:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176884.218950:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176884.218952:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176884.218954:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176884.218956:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176884.218958:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856310144, portal 4
00000100:00000001:1.0:1786176884.218960:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176884.218961:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00000040:1.0:1786176884.218964:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.218969:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176884.218970:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176884.218971:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176884.218973:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856310144, offset 0
00000400:00000200:1.0:1786176884.218975:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176884.218978:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176884.218980:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176884.218982:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176884.218984:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176884.218986:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176884.218987:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176884.218990:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254594:254594:256:4294967295] 192.168.202.110@tcp LPNI seq info [254583:254583:8:4294967295]
00000400:00000200:1.0:1786176884.218993:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176884.218997:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176884.219000:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176884.219002:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176884.219005:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176884.219007:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786176884.219009:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176884.219011:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1786176884.219014:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176884.219015:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176884.219018:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176884.219020:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176884.219022:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176884.219023:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176884.219025:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176884.219027:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.219032:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176884.219034:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176909 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.219038:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.219039:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176884.219041:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176884.219043:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176884.219045:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176884.219046:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301779987
00000100:00000001:1.0:1786176884.219047:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301783757
00000100:00000001:1.0:1786176884.219049:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301779987
00000100:00000001:1.0:1786176884.219050:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301783757
00000100:00000001:1.0:1786176884.219052:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176884.219053:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.219055:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176884.219057:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176884/real 1786176884]  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176884.219062:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176884.219063:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176884.219064:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176884.219065:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176884.219066:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176884.219067:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176884.219069:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176884.219075:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.219079:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176884.219080:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.219081:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176884.219081:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176884.219084:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.219087:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176884.219090:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.219094:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176884.219095:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.219096:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176884.219097:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.219099:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176884.219100:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176884.219101:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176884.219104:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176884.219105:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176884.219108:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176884.219113:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856310144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176884.219115:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.219116:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176884.219118:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856310016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.219122:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176884.219123:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176884.219124:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176884.219125:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176884.219127:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176884.219128:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176884.219130:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176884.219132:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786176884.219135:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176884.219137:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176884.219138:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:1.0:1786176884.219140:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176884.219142:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.219143:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176884.219147:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856310144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176909 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176884.219154:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176884.219156:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176884.219157:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176884.219159:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176884.219161:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176884.219162:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176884.219165:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176884.219167:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176884.219170:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176884.219171:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176884.219173:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:1.0:1786176884.219175:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176884.219176:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.219178:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176884.219182:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176884.219184:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176884.219186:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176884.219187:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176884.792612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176884.792618:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176884.792623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176884.792624:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176884.792627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176884.792628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176884.792682:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176884.792687:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176884.792691:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176885.240635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176885.240644:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176885.240647:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176885.240652:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176885.240656:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176885.240661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176885.240665:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176885.240669:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176885.240672:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176885.816606:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176885.816611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176885.816615:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176885.816616:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176885.816619:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176885.816620:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176885.816675:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176885.816679:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176885.816682:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176886.264661:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176886.264671:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176886.264673:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176886.264678:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176886.264682:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176886.264685:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176886.264688:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176886.264690:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176886.264693:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176886.840599:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176886.840606:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176886.840612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176886.840614:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176886.840617:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176886.840618:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176886.840711:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176886.840718:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176886.840723:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176887.288641:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176887.288650:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176887.288653:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176887.288658:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176887.288662:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176887.288666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176887.288669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176887.288671:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176887.288674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176887.864630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176887.864635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176887.864639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176887.864640:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176887.864643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176887.864644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176887.864708:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176887.864711:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176887.864714:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786176888.056605:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301783757, at 4301783808
00000100:00000010:1.0:1786176888.056638:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1240.
00000100:00000040:0.0:1786176888.056641:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301783757, at 4301783808
00000100:00000010:0.0:1786176888.056649:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b18a0.
00000100:00000001:2.0:1786176888.313773:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176888.313780:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176888.313781:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176888.313786:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176888.313789:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176888.313792:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176888.313794:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176888.313797:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176888.313799:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176888.888568:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176888.888574:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176888.888581:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176888.888582:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176888.888586:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176888.888587:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176888.888638:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176888.888642:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176888.888646:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176889.336625:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176889.336632:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176889.336634:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176889.336639:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176889.336645:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176889.336650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786176889.336654:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786176889.336660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176889.336665:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786176889.336667:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176889.336671:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176889.336673:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176889.336680:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176889.336686:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176889.336696:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7112
00000400:00000200:0.0:1786176889.336707:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176889.336715:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176889.336719:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176889.336721:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.336724:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176889.336727:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.336729:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176889.336733:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176889.336735:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786176889.336753:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176889.336757:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786176889.336766:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786176889.336769:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786176889.336776:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.336779:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176889.336781:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.336783:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176889.336786:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786176889.336790:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176889.336793:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176889.336797:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176889.336802:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176889.336807:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176889.336810:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786176889.336813:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176889.336826:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41aa00.
00000020:00000040:0.0:1786176889.336831:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786176889.336836:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176889.336838:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176889.336842:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176889.336846:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176889.336851:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176889.336862:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000001:0.0:1786176889.336869:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176889.336881:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176889.336894:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176889.336900:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176889.336906:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da0255a4] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176889.336933:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176889.336937:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176889.336941:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786176889.336943:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176889.336945:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176889.336949:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176889.336951:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7112
00000400:00000200:0.0:1786176889.336956:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176889.336960:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176889.336963:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176889.336967:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.336970:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176889.336972:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.336975:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176889.336978:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176889.336979:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786176889.336982:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176889.336982:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1786176889.336984:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786176889.336989:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176889.336990:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1786176889.336994:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176889.336995:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786176889.337002:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337007:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786176889.337008:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176889.337011:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337018:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176889.337023:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786176889.337028:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337032:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.337032:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1786176889.337037:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176889.337037:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176889.337044:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176889.337045:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176889.337048:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176889.337048:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176889.337053:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786176889.337055:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1786176889.337058:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786176889.337063:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176889.337064:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856310272:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786176889.337065:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176889.337070:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41ad80.
00000020:00000040:0.0:1786176889.337074:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786176889.337074:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176889.337079:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176889.337080:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176889.337082:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786176889.337082:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176889.337085:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176889.337088:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176889.337090:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176889.337091:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5800.
02000000:00000001:1.0:1786176889.337095:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1786176889.337096:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176889.337103:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176889.337109:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176889.337110:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176889.337113:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034e2c676] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1786176889.337116:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
00000100:00000001:0.0:1786176889.337120:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176889.337123:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:1.0:1786176889.337126:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176889.337128:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786176889.337131:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176889.337133:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176889.337136:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176889.337139:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7112
00000400:00000200:1.0:1786176889.337141:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176889.337145:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176889.337148:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000010:1.0:1786176889.337149:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00080000:0.0:1786176889.337150:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176889.337156:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337158:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176889.337159:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337161:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000010:1.0:1786176889.337162:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000020:00000001:0.0:1786176889.337164:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176889.337167:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786176889.337169:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176889.337171:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000200:1.0:1786176889.337173:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856310272, portal 25
00000020:00000001:0.0:1786176889.337174:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:1.0:1786176889.337179:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000020:00000001:0.0:1786176889.337180:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:1.0:1786176889.337184:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761665536 : -114547947886080 : ffff97d1ba41aa00)
00000100:00000040:0.0:1786176889.337187:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337192:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786176889.337196:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176889.337197:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337204:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176889.337210:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786176889.337216:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176889.337217:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1786176889.337222:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176889.337224:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786176889.337225:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176889.337231:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:1.0:1786176889.337231:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
02000000:00000001:0.0:1786176889.337237:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176889.337239:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856310272, offset 0
00000100:00000001:0.0:1786176889.337240:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176889.337242:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786176889.337246:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176889.337247:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176889.337251:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b480.
00000400:00000200:1.0:1786176889.337252:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:0.0:1786176889.337254:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786176889.337257:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176889.337259:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176889.337260:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176889.337261:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176889.337262:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176889.337266:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000400:00000200:1.0:1786176889.337267:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176889.337275:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176889.337282:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176889.337287:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176889.337292:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176889.337295:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176889.337301:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254595:254595:256:4294967295] 192.168.202.110@tcp LPNI seq info [254584:254584:8:4294967295]
00000400:00000200:1.0:1786176889.337306:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176889.337318:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176889.337325:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176889.337332:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176889.337341:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176889.337346:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786176889.337351:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786176889.337369:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176889.337383:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176889.337392:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176889.337396:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176889.337399:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001215e9e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176889.337408:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176889.337411:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176889.337414:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786176889.337417:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176889.337421:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176889.337425:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.1:1786176889.337425:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00080000:0.0:1786176889.337429:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7112
00000800:00000100:1.0:1786176889.337437:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176889.337439:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000010:1.0:1786176889.337442:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:0.0:1786176889.337447:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176889.337450:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786176889.337451:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1786176889.337455:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337459:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176889.337460:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:0.0:1786176889.337465:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337468:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176889.337470:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000020:00000001:0.0:1786176889.337471:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176889.337474:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000400:00000010:1.0:1786176889.337474:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000020:00000001:0.0:1786176889.337479:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176889.337481:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:0.0:1786176889.337483:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786176889.337485:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000200:1.0:1786176889.337488:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786176889.337489:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786176889.337498:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337502:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176889.337506:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176889.337508:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786176889.337509:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176889.337540:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786176889.337545:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:1.0:1786176889.337546:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176889.337549:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176889.337554:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176889.337559:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.337560:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176889.337567:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176889.337573:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786176889.337573:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176889.337576:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301788878
02000000:00000001:0.0:1786176889.337577:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176889.337581:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176889.337585:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786176889.337587:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176889.337589:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786176889.337591:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1240.
00000100:00000010:0.0:1786176889.337595:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b100.
00000020:00000040:0.0:1786176889.337600:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176889.337600:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:0.0:1786176889.337605:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176889.337608:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176889.337609:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176889.337611:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176889.337611:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301788878
02000000:00000001:0.0:1786176889.337615:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786176889.337619:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1f60.
02000000:00000010:0.0:1786176889.337623:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7800.
00000100:00000001:1.0:1786176889.337625:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786176889.337627:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.337631:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:0.0:1786176889.337633:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176889.337636:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.337641:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1786176889.337645:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176889.337649:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176889.337649:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176889/real 1786176889]  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176889.337653:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000fa632fb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176889.337659:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176889.337662:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176889.337670:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786176889.337674:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176889.337677:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000001:1.0:1786176889.337678:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1786176889.337680:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000001:1.0:1786176889.337682:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:0.0:1786176889.337683:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7112
00000400:00000010:1.0:1786176889.337686:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:0.0:1786176889.337691:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176889.337692:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000200:0.0:1786176889.337696:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176889.337698:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:1.0:1786176889.337700:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000040:0.0:1786176889.337701:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337705:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176889.337705:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:0.0:1786176889.337710:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337714:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:1.0:1786176889.337714:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786176889.337718:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176889.337720:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786176889.337724:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176889.337727:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786176889.337731:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000200:1.0:1786176889.337731:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786176889.337734:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786176889.337758:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337760:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176889.337762:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176889.337764:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176889.337770:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:1.0:1786176889.337772:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176889.337776:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786176889.337779:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176889.337779:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176889.337782:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176889.337786:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176889.337789:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786176889.337791:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176889.337797:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176889.337798:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786176889.337804:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176889.337807:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176889.337809:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176889.337813:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786176889.337818:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000008:00000001:0.0:1786176889.337819:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176889.337823:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.337824:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.337829:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176889.337830:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:0.0:1786176889.337831:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419180.
00000100:00000001:1.0:1786176889.337832:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000040:0.0:1786176889.337837:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176889.337837:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:0.0:1786176889.337843:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00080000:1.0:1786176889.337844:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:0.0:1786176889.337847:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786176889.337850:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176889.337851:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176889.337855:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176889.337855:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:0.0:1786176889.337857:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176889.337862:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786176889.337863:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000100:00000001:0.0:1786176889.337869:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176889.337873:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856310272:192.168.202.110@tcp:250:kworker.0
00000100:00100000:0.0:1786176889.337877:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176889.337880:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.337885:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786176889.337889:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1786176889.337890:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41aa00 x1872934856310272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176889.337892:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176889.337901:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005a89d8ab] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.337902:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176889.337905:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176889.337909:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
00000100:00000001:0.0:1786176889.337911:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176889.337913:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176889.337916:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000040:0.0:1786176889.337917:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7122)
00000020:00000040:1.0:1786176889.337918:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176889.337922:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176889.337925:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:1.0:1786176889.337928:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176889.337929:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176889.337930:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41aa00.
00000100:00000001:1.0:1786176889.337933:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176889.337934:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.337936:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.337956:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176889.337958:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.337960:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176889.337962:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176889.337963:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176889.337967:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.337972:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176889.337974:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176889.337976:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176889.337978:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176889.337979:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176889.337982:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856310400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176889.337984:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176889.337987:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176889.337989:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176889.337990:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176889.337994:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176889.337995:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176889.337999:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176889.338002:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176889.338005:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176889.338010:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856310400, portal 10
00000100:00000001:1.0:1786176889.338011:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176889.338013:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761666432 : -114547947885184 : ffff97d1ba41ad80)
00000100:00000040:1.0:1786176889.338018:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338023:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176889.338025:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176889.338026:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176889.338028:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856310400, offset 0
00000400:00000200:1.0:1786176889.338032:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176889.338038:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176889.338042:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176889.338046:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176889.338050:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176889.338053:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176889.338055:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176889.338058:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254596:254596:256:4294967295] 192.168.202.110@tcp LPNI seq info [254585:254585:8:4294967295]
00000400:00000200:1.0:1786176889.338061:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176889.338067:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176889.338071:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176889.338074:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176889.338079:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176889.338081:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786176889.338084:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176889.338086:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1786176889.338089:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176889.338091:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176889.338093:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176889.338095:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176889.338097:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176889.338098:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176889.338100:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176889.338102:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338107:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176889.338109:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338114:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.338115:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176889.338117:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176889.338121:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176889.338123:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176889.338125:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301785098
00000100:00000001:1.0:1786176889.338127:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301788878
00000100:00000001:1.0:1786176889.338128:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301785098
00000100:00000001:1.0:1786176889.338131:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301788878
00000100:00000001:1.0:1786176889.338132:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176889.338134:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.338135:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176889.338138:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176889/real 1786176889]  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176889.338142:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176889.338143:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176889.338144:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176889.338146:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176889.338146:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176889.338148:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176889.338149:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176889.338155:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338159:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176889.338160:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338161:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176889.338162:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176889.338164:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338168:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176889.338170:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338174:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176889.338176:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338177:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176889.338177:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338179:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176889.338181:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176889.338192:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176889.338195:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176889.338196:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176889.338199:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176889.338204:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856310400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176889.338207:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176889.338209:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338213:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176889.338214:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176889.338216:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176889.338217:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176889.338218:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176889.338220:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856310528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176889.338222:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176889.338223:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176889.338224:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176889.338225:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176889.338227:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176889.338229:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176889.338231:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176889.338232:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176889.338234:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176889.338236:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856310528, portal 10
00000100:00000001:1.0:1786176889.338237:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176889.338238:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761668224 : -114547947883392 : ffff97d1ba41b480)
00000100:00000040:1.0:1786176889.338242:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338247:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176889.338248:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176889.338250:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176889.338251:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856310528, offset 0
00000400:00000200:1.0:1786176889.338254:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176889.338256:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176889.338258:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176889.338260:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176889.338262:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176889.338263:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176889.338265:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176889.338267:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254597:254597:256:4294967295] 192.168.202.110@tcp LPNI seq info [254586:254586:8:4294967295]
00000400:00000200:1.0:1786176889.338270:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176889.338274:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176889.338277:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176889.338279:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176889.338281:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176889.338283:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786176889.338286:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176889.338287:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1786176889.338290:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176889.338292:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176889.338293:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176889.338296:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176889.338298:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176889.338299:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176889.338300:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176889.338303:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338308:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176889.338310:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338318:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.338319:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176889.338321:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176889.338323:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176889.338325:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176889.338326:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301785223
00000100:00000001:1.0:1786176889.338328:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301788878
00000100:00000001:1.0:1786176889.338329:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301785223
00000100:00000001:1.0:1786176889.338331:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301788878
00000100:00000001:1.0:1786176889.338332:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176889.338333:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.338334:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176889.338336:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176889/real 1786176889]  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176889.338341:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176889.338342:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176889.338343:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176889.338344:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176889.338345:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176889.338346:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176889.338348:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176889.338353:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338358:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176889.338359:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338360:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176889.338361:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176889.338363:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338368:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176889.338370:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338374:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176889.338376:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338376:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176889.338377:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338378:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176889.338379:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176889.338381:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176889.338383:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176889.338384:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176889.338386:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176889.338391:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856310528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176889.338393:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176889.338395:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338399:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176889.338401:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176889.338402:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176889.338403:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176889.338405:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176889.338407:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856310656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176889.338408:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176889.338409:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176889.338410:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176889.338412:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176889.338413:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176889.338415:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176889.338417:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176889.338418:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176889.338420:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176889.338422:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856310656, portal 4
00000100:00000001:1.0:1786176889.338423:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176889.338424:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761667328 : -114547947884288 : ffff97d1ba41b100)
00000100:00000040:1.0:1786176889.338428:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338432:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176889.338434:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176889.338435:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176889.338436:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856310656, offset 0
00000400:00000200:1.0:1786176889.338439:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176889.338441:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176889.338443:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176889.338445:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176889.338446:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176889.338448:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176889.338450:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176889.338452:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254598:254598:256:4294967295] 192.168.202.110@tcp LPNI seq info [254587:254587:8:4294967295]
00000400:00000200:1.0:1786176889.338454:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176889.338458:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176889.338461:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176889.338463:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176889.338465:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176889.338466:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786176889.338469:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176889.338471:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1786176889.338473:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176889.338475:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176889.338477:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176889.338479:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176889.338481:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176889.338482:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176889.338484:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176889.338486:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338491:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176889.338493:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338497:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.338498:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176889.338500:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176889.338502:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176889.338504:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176889.338505:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301785098
00000100:00000001:1.0:1786176889.338506:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301788878
00000100:00000001:1.0:1786176889.338508:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301785098
00000100:00000001:1.0:1786176889.338524:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301788878
00000100:00000001:1.0:1786176889.338527:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176889.338528:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.338530:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176889.338533:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176889/real 1786176889]  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176889.338538:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176889.338539:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176889.338540:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176889.338542:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176889.338543:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176889.338544:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176889.338546:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176889.338552:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338556:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176889.338556:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338557:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176889.338558:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176889.338560:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338564:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176889.338567:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338571:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176889.338572:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338573:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176889.338574:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338576:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176889.338578:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176889.338581:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176889.338584:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176889.338586:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176889.338590:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176889.338598:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856310656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176889.338602:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176889.338605:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338613:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176889.338615:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176889.338617:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176889.338619:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176889.338621:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176889.338624:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856310784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176889.338628:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176889.338629:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176889.338631:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176889.338633:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176889.338636:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176889.338637:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176889.338639:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176889.338641:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176889.338643:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176889.338645:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856310784, portal 4
00000100:00000001:1.0:1786176889.338646:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176889.338647:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761659264 : -114547947892352 : ffff97d1ba419180)
00000100:00000040:1.0:1786176889.338652:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338656:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176889.338658:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176889.338659:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176889.338661:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856310784, offset 0
00000400:00000200:1.0:1786176889.338663:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176889.338666:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176889.338668:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176889.338670:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176889.338672:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176889.338673:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176889.338675:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176889.338677:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254599:254599:256:4294967295] 192.168.202.110@tcp LPNI seq info [254588:254588:8:4294967295]
00000400:00000200:1.0:1786176889.338680:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176889.338684:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176889.338687:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176889.338689:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176889.338691:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176889.338693:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786176889.338696:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176889.338698:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1786176889.338700:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176889.338702:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176889.338704:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176889.338706:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176889.338708:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176889.338709:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176889.338710:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176889.338713:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338718:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176889.338720:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338724:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.338725:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176889.338727:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176889.338729:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176889.338731:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176889.338732:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301785107
00000100:00000001:1.0:1786176889.338734:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301788878
00000100:00000001:1.0:1786176889.338756:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301785107
00000100:00000001:1.0:1786176889.338759:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301788878
00000100:00000001:1.0:1786176889.338760:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176889.338762:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.338764:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176889.338767:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176889/real 1786176889]  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176889.338772:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176889.338773:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176889.338774:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176889.338776:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176889.338777:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176889.338779:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176889.338781:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176889.338786:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338790:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176889.338791:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338792:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176889.338793:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176889.338795:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338799:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176889.338801:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338805:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176889.338807:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338808:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176889.338809:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338810:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176889.338812:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176889.338813:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176889.338816:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176889.338818:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176889.338822:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176889.338830:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856310784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176889.338834:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338836:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176889.338839:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41ad80 x1872934856310400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338846:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176889.338848:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176889.338850:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176889.338852:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176889.338854:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176889.338855:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176889.338857:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176889.338859:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5800.
02000000:00000001:1.0:1786176889.338862:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176889.338864:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176889.338865:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41ad80.
00000100:00000001:1.0:1786176889.338868:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176889.338869:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338871:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176889.338875:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b480 x1872934856310528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338882:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176889.338883:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176889.338885:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176889.338887:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176889.338888:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176889.338890:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176889.338891:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176889.338893:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:1.0:1786176889.338895:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176889.338897:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176889.338898:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b480.
00000100:00000001:1.0:1786176889.338900:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176889.338902:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338904:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176889.338907:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b100 x1872934856310656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338914:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176889.338916:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176889.338917:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176889.338919:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176889.338921:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176889.338922:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176889.338924:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176889.338926:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7800.
02000000:00000001:1.0:1786176889.338928:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176889.338930:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176889.338931:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b100.
00000100:00000001:1.0:1786176889.338933:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176889.338934:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338936:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176889.338939:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419180 x1872934856310784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176889.338945:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176889.338946:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176889.338947:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176889.338950:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176889.338951:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176889.338952:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176889.338953:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176889.338954:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:1.0:1786176889.338955:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176889.338956:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176889.338957:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419180.
00000100:00000001:1.0:1786176889.338959:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176889.338960:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338961:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176889.338964:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176889.338965:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176889.338966:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176889.338968:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176889.912593:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176889.912602:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176889.912604:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176889.912607:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176889.912618:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176889.912621:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176889.912622:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176889.912624:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176889.912627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176889.912629:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176889.912629:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176889.912631:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176889.912632:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176889.912633:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176889.912634:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176889.912636:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176889.912638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176889.912639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176889.912699:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176889.912704:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176889.912708:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176890.360632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176890.360640:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176890.360642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176890.360648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176890.360651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176890.360655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176890.360658:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176890.360661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176890.360664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176890.936625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176890.936633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176890.936640:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176890.936641:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176890.936645:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176890.936647:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176890.936863:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176890.936869:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176890.936872:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176891.384607:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176891.384613:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176891.384616:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176891.384621:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176891.384623:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176891.384626:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176891.384629:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176891.384631:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176891.384632:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176891.960618:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176891.960623:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176891.960628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176891.960629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176891.960633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176891.960634:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176891.960677:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176891.960681:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176891.960684:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176892.408588:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176892.408594:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176892.408596:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176892.408601:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176892.408604:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176892.408606:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176892.408609:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176892.408611:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176892.408613:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176892.984655:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176892.984663:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176892.984669:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176892.984670:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176892.984673:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176892.984676:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176892.984678:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176892.984682:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176892.984685:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176892.984687:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176892.984689:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176892.984692:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176892.984770:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176892.984774:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176892.984777:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786176893.176617:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301788878, at 4301788928
00000100:00000040:0.0:1786176893.176621:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301788878, at 4301788928
00000100:00000010:1.0:1786176893.176635:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1f60.
00000100:00000010:0.0:1786176893.176637:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1240.
00000100:00000001:2.0:1786176893.432584:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176893.432591:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176893.432593:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176893.432599:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176893.432604:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176893.432608:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176893.432611:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176893.432614:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176893.432626:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176894.008632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176894.008638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176894.008643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176894.008644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176894.008647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176894.008648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176894.008722:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176894.008726:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176894.008729:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176894.456580:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176894.456588:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176894.456591:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176894.456595:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1786176894.456597:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176894.456603:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1786176894.456604:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176894.456609:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176894.456609:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176894.456613:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176894.456614:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176894.456618:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786176894.456620:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176894.456625:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176894.456628:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7117
00000400:00000200:3.0:1786176894.456638:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176894.456644:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176894.456648:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176894.456651:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.456655:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176894.456657:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.456659:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176894.456663:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176894.456665:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176894.456668:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176894.456671:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176894.456673:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176894.456676:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176894.456680:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.456682:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176894.456684:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.456711:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176894.456714:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176894.456718:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176894.456721:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176894.456724:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176894.456727:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176894.456730:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176894.456732:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176894.456734:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176894.456743:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69dc00.
00000020:00000040:3.0:1786176894.456746:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176894.456750:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176894.456752:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176894.456755:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176894.456758:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176894.456762:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176894.456769:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:3.0:1786176894.456775:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176894.456784:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176894.456795:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176894.456799:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176894.456803:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097fe80e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176894.456823:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176894.456827:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176894.456830:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176894.456831:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176894.456834:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176894.456837:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176894.456839:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7117
00000400:00000200:3.0:1786176894.456843:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176894.456847:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176894.456849:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176894.456853:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.456856:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176894.456858:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.456861:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176894.456863:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176894.456864:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1786176894.456865:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:1.0:1786176894.456869:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1786176894.456869:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176894.456874:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1786176894.456874:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786176894.456878:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176894.456882:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:1.0:1786176894.456884:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176894.456888:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.456894:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176894.456899:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176894.456900:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176894.456903:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176894.456904:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786176894.456910:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
02000000:00000001:1.0:1786176894.456911:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176894.456915:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1786176894.456916:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:1.0:1786176894.456920:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176894.456922:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176894.456926:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:1.0:1786176894.456927:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856310912:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176894.456931:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176894.456933:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176894.456934:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176894.456937:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176894.456938:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176894.456942:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1786176894.456942:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176894.456948:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000010:1.0:1786176894.456949:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
00000008:00000001:3.0:1786176894.456953:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176894.456954:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176894.456959:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176894.456964:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:3.0:1786176894.456967:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e680.
00000400:00000010:1.0:1786176894.456969:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000020:00000040:3.0:1786176894.456970:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176894.456975:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176894.456976:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176894.456977:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000001:3.0:1786176894.456978:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176894.456981:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176894.456983:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176894.456985:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856310912, portal 25
00000100:00000001:1.0:1786176894.456989:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
02000000:00000010:3.0:1786176894.456989:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:1.0:1786176894.456992:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764299776 : -114547945251840 : ffff97d1ba69dc00)
00000100:00000001:3.0:1786176894.456996:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176894.456999:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176894.457005:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176894.457010:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176894.457017:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176894.457022:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000001:3.0:1786176894.457023:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1786176894.457025:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856310912, offset 0
00000100:00000001:3.0:1786176894.457027:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176894.457034:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786176894.457034:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1839131] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176894.457046:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176894.457047:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176894.457051:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786176894.457053:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:3.0:1786176894.457056:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786176894.457059:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1786176894.457060:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1786176894.457061:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176894.457065:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786176894.457066:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:3.0:1786176894.457071:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7117
00000400:00000200:1.0:1786176894.457072:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176894.457077:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786176894.457080:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176894.457085:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254600:254600:256:4294967295] 192.168.202.110@tcp LPNI seq info [254589:254589:8:4294967295]
00000400:00000200:3.0:1786176894.457088:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176894.457093:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786176894.457094:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1786176894.457099:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.457102:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176894.457108:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.457110:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176894.457112:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176894.457114:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786176894.457117:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786176894.457121:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:3.0:1786176894.457122:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786176894.457127:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786176894.457132:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:1.0:1786176894.457133:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786176894.457139:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000010:1.0:1786176894.457140:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000100:00000001:3.0:1786176894.457143:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176894.457146:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:1.0:1786176894.457151:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786176894.457151:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:1.0:1786176894.457157:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000020:00000040:3.0:1786176894.457158:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786176894.457165:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:1.1:1786176894.457168:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786176894.457170:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176894.457173:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176894.457175:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176894.457177:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176894.457179:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176894.457180:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176894.457182:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176894.457184:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176894.457187:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ed80.
00000020:00000040:3.0:1786176894.457189:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176894.457191:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176894.457192:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176894.457194:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176894.457195:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176894.457196:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176894.457200:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:3.0:1786176894.457203:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176894.457207:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.1:1786176894.457212:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1786176894.457223:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176894.457225:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000010:1.0:1786176894.457226:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000100:00000001:3.0:1786176894.457230:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176894.457234:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1786176894.457237:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ccd6c6d2] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176894.457242:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176894.457249:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176894.457250:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176894.457254:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1786176894.457259:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176894.457260:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176894.457264:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176894.457272:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786176894.457276:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786176894.457277:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1786176894.457278:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:1.0:1786176894.457283:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.457289:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.457290:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176894.457295:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176894.457297:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176894.457299:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301793997
00000100:00000001:3.0:1786176894.457301:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000010:1.0:1786176894.457304:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1f60.
00000100:00080000:3.0:1786176894.457307:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7117
00000100:00000001:1.0:1786176894.457312:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000400:00000200:3.0:1786176894.457313:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176894.457320:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301793997
00000400:00000200:3.0:1786176894.457320:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176894.457324:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000010:1.0:1786176894.457325:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b14e0.
00000100:00000001:1.0:1786176894.457330:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:3.0:1786176894.457331:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176894.457333:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176894.457335:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176894.457336:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.457339:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:3.0:1786176894.457340:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.457345:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00100000:1.0:1786176894.457346:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176894/real 1786176894]  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176894.457355:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:3.0:1786176894.457355:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000001:1.0:1786176894.457358:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:3.0:1786176894.457359:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000400:00000010:1.0:1786176894.457361:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000020:00000001:3.0:1786176894.457362:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786176894.457364:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176894.457367:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176894.457369:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:3.0:1786176894.457369:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786176894.457374:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000200:1.0:1786176894.457375:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786176894.457381:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786176894.457389:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:1.0:1786176894.457390:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176894.457394:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176894.457400:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176894.457401:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176894.457405:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176894.457405:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176894.457412:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:3.0:1786176894.457412:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:1.0:1786176894.457415:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176894.457419:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:1.0:1786176894.457422:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176894.457423:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176894.457429:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176894.457433:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786176894.457434:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1786176894.457439:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176894.457439:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176894.457444:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176894.457448:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786176894.457449:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000008:00000001:3.0:1786176894.457452:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.457453:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176894.457456:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.457457:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176894.457460:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.457463:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000010:3.0:1786176894.457464:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c380.
00000100:00000001:1.0:1786176894.457465:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:3.0:1786176894.457469:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00080000:1.0:1786176894.457470:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:3.0:1786176894.457476:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00080000:1.0:1786176894.457477:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
02000000:00000001:3.0:1786176894.457479:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.457480:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1786176894.457482:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176894.457485:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:1.0:1786176894.457486:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176894.457487:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176894.457495:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00100000:1.0:1786176894.457496:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856310912:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786176894.457500:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.457502:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.457504:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00100000:3.0:1786176894.457508:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1786176894.457542:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69dc00 x1872934856310912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.457568:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000001:3.0:1786176894.457568:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1786176894.457571:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:3.0:1786176894.457572:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176894.457575:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176894.457580:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000040:3.0:1786176894.457580:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000833bf1c9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1786176894.457582:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176894.457586:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176894.457591:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176894.457594:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
00000100:00000001:3.0:1786176894.457597:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176894.457599:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00080000:3.0:1786176894.457601:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:1.0:1786176894.457602:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00080000:3.0:1786176894.457608:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786176894.457609:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176894.457611:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000010:1.0:1786176894.457613:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69dc00.
00000100:00000001:3.0:1786176894.457614:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176894.457620:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00080000:3.0:1786176894.457620:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7117
00000100:00000001:1.0:1786176894.457624:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.457627:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1786176894.457628:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176894.457633:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176894.457634:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:3.0:1786176894.457636:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176894.457637:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.457641:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1786176894.457642:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176894.457643:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176894.457647:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1786176894.457648:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786176894.457653:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176894.457654:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.457659:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176894.457664:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176894.457667:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:1.0:1786176894.457669:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000001:3.0:1786176894.457670:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176894.457671:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786176894.457674:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
02000000:00000001:1.0:1786176894.457675:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176894.457679:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:3.0:1786176894.457679:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
02000000:00000001:1.0:1786176894.457685:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176894.457698:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786176894.457703:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.457707:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:1.0:1786176894.457711:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311040:192.168.202.110@tcp:38:kworker.0
00000100:00000040:3.0:1786176894.457712:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176894.457718:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176894.457720:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:3.0:1786176894.457722:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
02000000:00000001:1.0:1786176894.457725:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:3.0:1786176894.457726:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:1.0:1786176894.457729:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176894.457730:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176894.457732:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176894.457735:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176894.457737:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176894.457742:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1786176894.457742:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176894.457747:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000010:1.0:1786176894.457748:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
00000008:00000001:3.0:1786176894.457750:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176894.457754:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176894.457754:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176894.457763:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d880.
00000400:00000200:1.0:1786176894.457765:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:3.0:1786176894.457767:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176894.457771:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:1.0:1786176894.457772:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
02000000:00000001:3.0:1786176894.457773:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176894.457779:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176894.457780:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:1.0:1786176894.457781:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
02000000:00000001:3.0:1786176894.457783:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176894.457788:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856311040, portal 10
02000000:00000010:3.0:1786176894.457790:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:1.0:1786176894.457793:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176894.457796:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.457797:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00100000:3.0:1786176894.457805:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1786176894.457810:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176894.457821:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176894.457825:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.457829:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176894.457834:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097f9b8db] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786176894.457837:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176894.457843:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000001:3.0:1786176894.457847:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176894.457848:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856311040, offset 0
00000100:00000040:3.0:1786176894.457853:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7127)
00000400:00000200:1.0:1786176894.457856:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176894.457861:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176894.457865:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176894.457868:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176894.457871:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176894.457873:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176894.457876:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176894.457879:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254601:254601:256:4294967295] 192.168.202.110@tcp LPNI seq info [254590:254590:8:4294967295]
00000400:00000200:1.0:1786176894.457882:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176894.457887:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176894.457890:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176894.457893:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176894.457895:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176894.457897:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786176894.457901:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176894.457902:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1786176894.457905:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176894.457906:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176894.457909:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176894.457911:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176894.457913:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176894.457914:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176894.457916:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176894.457919:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.457931:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176894.457934:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.457941:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.457943:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176894.457946:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176894.457950:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176894.457954:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176894.457957:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301790218
00000100:00000001:1.0:1786176894.457960:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301793997
00000100:00000001:1.0:1786176894.457962:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301790218
00000100:00000001:1.0:1786176894.457964:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301793997
00000100:00000001:1.0:1786176894.457967:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176894.457969:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.457971:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176894.457975:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176894/real 1786176894]  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176894.457982:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176894.457984:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176894.457985:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176894.457988:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176894.457989:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176894.457991:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176894.457994:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176894.458002:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458009:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176894.458011:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458013:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176894.458014:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176894.458018:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458024:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176894.458027:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458034:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176894.458036:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458038:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176894.458039:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458041:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176894.458043:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176894.458045:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176894.458048:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176894.458050:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176894.458055:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176894.458062:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176894.458066:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176894.458069:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458076:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176894.458077:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176894.458079:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176894.458081:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176894.458083:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176894.458106:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176894.458110:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176894.458112:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176894.458114:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176894.458116:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176894.458119:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176894.458121:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176894.458125:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176894.458128:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176894.458131:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176894.458134:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856311168, portal 10
00000100:00000001:1.0:1786176894.458136:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176894.458138:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:1.0:1786176894.458144:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458151:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176894.458154:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176894.458156:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176894.458158:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856311168, offset 0
00000400:00000200:1.0:1786176894.458162:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176894.458167:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176894.458171:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176894.458174:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176894.458177:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176894.458180:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176894.458182:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176894.458186:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254602:254602:256:4294967295] 192.168.202.110@tcp LPNI seq info [254591:254591:8:4294967295]
00000400:00000200:1.0:1786176894.458191:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176894.458197:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176894.458203:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176894.458206:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176894.458210:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176894.458213:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786176894.458216:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176894.458218:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1786176894.458220:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176894.458222:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176894.458224:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176894.458226:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176894.458229:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176894.458230:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176894.458232:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176894.458234:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458240:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176894.458242:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458247:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.458248:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176894.458250:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176894.458252:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176894.458254:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176894.458256:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301790343
00000100:00000001:1.0:1786176894.458257:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301793997
00000100:00000001:1.0:1786176894.458259:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301790343
00000100:00000001:1.0:1786176894.458260:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301793997
00000100:00000001:1.0:1786176894.458262:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176894.458263:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.458264:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176894.458267:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176894/real 1786176894]  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176894.458272:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176894.458273:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176894.458275:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176894.458276:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176894.458277:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176894.458279:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176894.458281:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176894.458286:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458290:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176894.458291:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458292:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176894.458293:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176894.458295:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458299:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176894.458302:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458306:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176894.458307:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458308:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176894.458309:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458310:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176894.458311:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176894.458313:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176894.458315:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176894.458316:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176894.458318:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176894.458327:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176894.458330:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176894.458332:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458336:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176894.458337:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176894.458338:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176894.458339:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176894.458340:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176894.458342:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176894.458344:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176894.458345:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176894.458346:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176894.458347:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176894.458349:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176894.458351:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176894.458353:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176894.458355:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176894.458357:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176894.458359:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856311296, portal 4
00000100:00000001:1.0:1786176894.458360:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176894.458361:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764293504 : -114547945258112 : ffff97d1ba69c380)
00000100:00000040:1.0:1786176894.458365:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458370:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176894.458371:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176894.458372:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176894.458374:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856311296, offset 0
00000400:00000200:1.0:1786176894.458376:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176894.458379:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176894.458382:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176894.458384:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176894.458385:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176894.458387:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176894.458389:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176894.458391:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254603:254603:256:4294967295] 192.168.202.110@tcp LPNI seq info [254592:254592:8:4294967295]
00000400:00000200:1.0:1786176894.458393:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176894.458397:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176894.458400:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176894.458402:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176894.458404:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176894.458406:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786176894.458409:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176894.458410:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1786176894.458413:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176894.458414:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176894.458416:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176894.458418:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176894.458420:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176894.458421:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176894.458423:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176894.458425:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458430:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176894.458432:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458436:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.458437:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176894.458438:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176894.458440:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176894.458442:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176894.458444:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301790218
00000100:00000001:1.0:1786176894.458445:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301793997
00000100:00000001:1.0:1786176894.458447:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301790218
00000100:00000001:1.0:1786176894.458448:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301793997
00000100:00000001:1.0:1786176894.458449:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176894.458450:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.458451:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176894.458454:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176894/real 1786176894]  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176894.458458:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176894.458459:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176894.458460:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176894.458461:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176894.458462:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176894.458463:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176894.458465:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176894.458471:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458474:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176894.458475:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458476:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176894.458477:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176894.458479:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458483:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176894.458485:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458489:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176894.458491:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458492:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176894.458493:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458494:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176894.458495:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176894.458497:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176894.458499:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176894.458500:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176894.458502:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176894.458507:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176894.458526:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458528:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176894.458530:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856311040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458534:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176894.458535:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176894.458537:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176894.458539:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176894.458540:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176894.458540:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176894.458542:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176894.458543:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:1.0:1786176894.458544:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176894.458545:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176894.458546:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:1.0:1786176894.458548:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176894.458549:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458550:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176894.458552:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856311168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458561:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176894.458562:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176894.458563:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176894.458564:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176894.458565:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176894.458566:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176894.458567:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176894.458568:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176894.458569:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176894.458570:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176894.458571:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:1.0:1786176894.458572:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176894.458573:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458574:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176894.458576:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c380 x1872934856311296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458579:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176894.458580:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176894.458581:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176894.458582:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176894.458583:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176894.458584:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176894.458585:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176894.458585:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786176894.458587:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176894.458587:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176894.458588:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c380.
00000100:00000001:1.0:1786176894.458589:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176894.458590:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458591:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458594:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176894.458595:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.458596:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176894.458597:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176894.458598:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176894.458600:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458604:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176894.458605:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176894.458607:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176894.458608:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176894.458609:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176894.458611:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176894.458613:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176894.458614:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176894.458615:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176894.458617:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176894.458619:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176894.458620:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176894.458622:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176894.458624:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176894.458626:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176894.458628:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856311424, portal 4
00000100:00000001:1.0:1786176894.458629:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176894.458630:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
00000100:00000040:1.0:1786176894.458634:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458638:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176894.458640:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176894.458641:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176894.458643:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856311424, offset 0
00000400:00000200:1.0:1786176894.458645:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176894.458647:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176894.458649:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176894.458651:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176894.458653:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176894.458654:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176894.458656:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176894.458658:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254604:254604:256:4294967295] 192.168.202.110@tcp LPNI seq info [254593:254593:8:4294967295]
00000400:00000200:1.0:1786176894.458661:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176894.458665:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176894.458668:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176894.458670:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176894.458672:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176894.458674:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1786176894.458677:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176894.458679:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1786176894.458682:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176894.458683:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176894.458685:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176894.458700:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176894.458703:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176894.458704:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176894.458705:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176894.458708:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458713:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176894.458715:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458720:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.458721:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176894.458723:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176894.458725:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176894.458727:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176894.458728:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301790227
00000100:00000001:1.0:1786176894.458730:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301793997
00000100:00000001:1.0:1786176894.458731:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301790227
00000100:00000001:1.0:1786176894.458733:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301793997
00000100:00000001:1.0:1786176894.458734:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176894.458736:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.458737:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176894.458739:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176894/real 1786176894]  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176894.458744:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176894.458745:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176894.458746:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176894.458747:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176894.458748:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176894.458750:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176894.458752:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176894.458757:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458762:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176894.458763:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458765:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176894.458766:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176894.458770:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458777:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176894.458781:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458788:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176894.458790:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458792:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176894.458793:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458795:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176894.458797:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176894.458799:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176894.458803:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176894.458805:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176894.458810:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176894.458818:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176894.458822:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458824:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176894.458827:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856311424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176894.458834:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176894.458836:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176894.458837:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176894.458840:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176894.458842:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176894.458843:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176894.458845:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176894.458847:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176894.458849:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176894.458850:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176894.458852:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:1.0:1786176894.458854:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176894.458856:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458858:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176894.458861:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176894.458863:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176894.458865:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176894.458866:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176895.032633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176895.032639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176895.032643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176895.032644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176895.032647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176895.032647:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176895.032745:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176895.032750:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176895.032754:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176895.480631:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176895.480640:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176895.480643:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176895.480650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176895.480654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176895.480658:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176895.480662:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176895.480665:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176895.480669:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176896.056615:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176896.056620:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176896.056624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176896.056625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176896.056627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176896.056628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176896.056689:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176896.056692:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176896.056695:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176896.504573:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176896.504582:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176896.504592:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176896.504599:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176896.504603:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176896.504607:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176896.504610:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176896.504613:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176896.504615:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176897.080599:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176897.080607:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176897.080613:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176897.080615:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176897.080618:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176897.080619:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176897.080676:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176897.080680:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176897.080682:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176897.528588:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176897.528595:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176897.528598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176897.528604:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176897.528608:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176897.528612:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176897.528615:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176897.528618:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176897.528621:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176898.104627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176898.104632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176898.104636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176898.104637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176898.104639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176898.104640:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176898.104700:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176898.104704:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176898.104708:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176898.296635:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301793997, at 4301794048
00000100:00000040:1.0:1786176898.296635:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301793997, at 4301794048
00000100:00000010:0.0:1786176898.296667:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1f60.
00000100:00000010:1.0:1786176898.296671:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b14e0.
00000100:00000001:2.0:1786176898.552611:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176898.552619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176898.552621:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176898.552627:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176898.552630:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176898.552634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176898.552638:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176898.552640:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176898.552643:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176899.128622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176899.128628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176899.128632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176899.128633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176899.128635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176899.128636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176899.128698:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176899.128701:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176899.128704:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786176899.576597:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786176899.576604:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1786176899.576607:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176899.576609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176899.576611:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1786176899.576615:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176899.576617:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1786176899.576621:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176899.576623:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786176899.576627:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176899.576633:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176899.576633:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7122
00000100:00000001:2.0:1786176899.576652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176899.576656:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176899.576660:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176899.576673:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176899.576681:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176899.576685:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176899.576688:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.576692:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176899.576695:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.576697:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176899.576701:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176899.576703:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176899.576706:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176899.576710:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176899.576713:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176899.576715:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176899.576718:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.576721:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176899.576723:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.576726:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176899.576729:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176899.576732:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176899.576734:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176899.576737:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176899.576749:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176899.576755:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176899.576757:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176899.576760:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176899.576767:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f800.
00000020:00000040:3.0:1786176899.576771:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176899.576776:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176899.576777:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176899.576780:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176899.576784:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176899.576788:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176899.576796:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786176899.576804:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176899.576813:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176899.576824:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176899.576829:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176899.576834:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4d339d9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176899.576856:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176899.576859:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176899.576863:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176899.576865:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176899.576866:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176899.576869:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176899.576872:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7122
00000400:00000200:3.0:1786176899.576876:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176899.576879:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176899.576882:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176899.576885:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.576888:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176899.576891:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.576894:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176899.576896:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1786176899.576897:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176899.576900:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:1.0:1786176899.576902:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1786176899.576904:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176899.576907:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1786176899.576908:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786176899.576912:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176899.576915:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:1.0:1786176899.576920:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176899.576920:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.576925:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176899.576929:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.576935:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176899.576940:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786176899.576942:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1786176899.576946:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176899.576947:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176899.576952:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176899.576954:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786176899.576955:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786176899.576959:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176899.576963:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176899.576964:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176899.576968:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176899.576972:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00100000:1.0:1786176899.576974:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856311552:192.168.202.110@tcp:250:kworker.0
00000008:00100000:3.0:1786176899.576977:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1786176899.576981:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:3.0:1786176899.576983:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176899.576987:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1786176899.576987:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176899.576992:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176899.576994:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69fb80.
02000000:00000001:1.0:1786176899.576997:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:3.0:1786176899.576999:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176899.577003:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176899.577004:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176899.577006:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
00000100:00000001:3.0:1786176899.577007:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786176899.577011:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176899.577011:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176899.577015:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176899.577022:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:3.0:1786176899.577022:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786176899.577029:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176899.577031:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00100000:3.0:1786176899.577038:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786176899.577041:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000001:3.0:1786176899.577051:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1786176899.577052:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856311552, portal 25
00000100:00000001:3.0:1786176899.577056:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.577058:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176899.577062:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
00000100:00000040:3.0:1786176899.577063:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014f55bbb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786176899.577075:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176899.577078:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176899.577083:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176899.577092:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1786176899.577096:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786176899.577096:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176899.577101:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:1.0:1786176899.577106:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176899.577107:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:1.0:1786176899.577112:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00080000:3.0:1786176899.577113:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7122
00000100:00000200:1.0:1786176899.577119:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856311552, offset 0
00000400:00000200:3.0:1786176899.577120:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176899.577127:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176899.577130:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786176899.577134:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786176899.577137:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.577140:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176899.577143:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.577147:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176899.577148:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1786176899.577153:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176899.577156:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786176899.577159:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786176899.577160:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:3.0:1786176899.577162:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786176899.577167:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786176899.577169:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:3.0:1786176899.577171:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786176899.577178:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:3.0:1786176899.577179:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.577183:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176899.577188:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1786176899.577189:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.577194:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176899.577197:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000040:3.0:1786176899.577198:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786176899.577206:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1786176899.577209:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254605:254605:256:4294967295] 192.168.202.110@tcp LPNI seq info [254594:254594:8:4294967295]
00000100:00000001:3.0:1786176899.577211:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176899.577216:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786176899.577220:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786176899.577220:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176899.577225:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176899.577227:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176899.577229:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176899.577234:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176899.577238:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176899.577241:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000010:3.0:1786176899.577243:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ca80.
00000020:00000040:3.0:1786176899.577249:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000800:00000200:1.0:1786176899.577254:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786176899.577255:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176899.577258:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176899.577261:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176899.577263:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000010:1.0:1786176899.577264:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
02000000:00000001:3.0:1786176899.577265:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176899.577274:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000800:00000200:1.0:1786176899.577277:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786176899.577279:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176899.577284:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00100000:3.0:1786176899.577288:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.1:1786176899.577296:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176899.577304:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786176899.577315:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176899.577318:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000010:1.0:1786176899.577320:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000100:00000001:3.0:1786176899.577322:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176899.577330:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1786176899.577330:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ffb0865] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176899.577343:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1786176899.577349:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176899.577352:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000100:00080000:3.0:1786176899.577353:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1786176899.577358:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00080000:3.0:1786176899.577360:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786176899.577364:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786176899.577366:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:3.0:1786176899.577366:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176899.577372:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:1.0:1786176899.577376:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176899.577377:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7122
00000400:00000200:3.0:1786176899.577386:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176899.577392:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176899.577397:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176899.577399:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176899.577401:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.577406:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786176899.577407:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176899.577411:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.577416:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176899.577421:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176899.577425:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:1.0:1786176899.577428:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176899.577428:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176899.577432:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:1.0:1786176899.577434:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:3.0:1786176899.577436:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786176899.577439:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786176899.577443:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176899.577444:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176899.577446:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176899.577450:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176899.577454:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176899.577455:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176899.577460:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301799117
00000020:00000040:3.0:1786176899.577461:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786176899.577467:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000010:1.0:1786176899.577471:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b14e0.
00000100:00000001:3.0:1786176899.577471:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176899.577476:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176899.577480:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176899.577483:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:3.0:1786176899.577483:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176899.577487:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176899.577491:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786176899.577493:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301799117
00000008:00000001:3.0:1786176899.577494:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176899.577498:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786176899.577500:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b15a0.
00000100:00000010:3.0:1786176899.577503:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f480.
00000100:00000001:1.0:1786176899.577506:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1786176899.577532:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1786176899.577536:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176899.577537:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.577543:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000001:3.0:1786176899.577544:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176899.577547:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176899.577551:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:1.0:1786176899.577552:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176899/real 1786176899]  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176899.577553:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176899.577557:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176899.577564:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000400:00000001:1.0:1786176899.577569:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1786176899.577570:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176899.577574:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176899.577579:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000100:00100000:3.0:1786176899.577580:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176899.577585:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176899.577589:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176899.577595:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:3.0:1786176899.577596:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176899.577600:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176899.577605:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176899.577610:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092aa1aa7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176899.577631:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176899.577632:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176899.577635:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176899.577659:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786176899.577662:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176899.577666:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176899.577667:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176899.577669:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176899.577672:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176899.577676:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7122
00000100:00000001:1.0:1786176899.577678:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176899.577683:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1786176899.577684:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:1.0:1786176899.577692:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786176899.577692:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176899.577697:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176899.577704:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.577709:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176899.577713:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176899.577714:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176899.577719:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176899.577722:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000040:1.0:1786176899.577724:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786176899.577724:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786176899.577729:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176899.577732:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786176899.577736:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786176899.577741:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176899.577745:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:3.0:1786176899.577748:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176899.577752:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176899.577753:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176899.577759:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:3.0:1786176899.577759:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176899.577764:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176899.577764:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176899.577770:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000040:3.0:1786176899.577770:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:1.0:1786176899.577775:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176899.577775:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176899.577780:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786176899.577783:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:3.0:1786176899.577784:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176899.577788:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176899.577792:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786176899.577793:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176899.577796:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176899.577798:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000008:00100000:3.0:1786176899.577800:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176899.577806:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176899.577809:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176899.577810:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176899.577818:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f100.
00000020:00000040:3.0:1786176899.577824:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00100000:1.0:1786176899.577828:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856311552:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1786176899.577830:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176899.577834:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.577838:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176899.577838:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176899.577842:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176899.577844:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1786176899.577845:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176899.577851:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856311552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1786176899.577852:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:3.0:1786176899.577858:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.577867:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00100000:3.0:1786176899.577868:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786176899.577873:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176899.577878:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
00000100:00000001:3.0:1786176899.577881:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1786176899.577884:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:3.0:1786176899.577885:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1786176899.577889:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000040:3.0:1786176899.577892:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1ef9f45] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1786176899.577893:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176899.577900:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176899.577905:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786176899.577910:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176899.577913:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000040:3.0:1786176899.577916:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7133)
02000000:00000001:1.0:1786176899.577917:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176899.577922:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000001:1.0:1786176899.577926:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176899.577927:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.577931:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.577938:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176899.577940:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.577942:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176899.577944:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176899.577946:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176899.577951:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.577958:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176899.577960:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176899.577963:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176899.577965:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176899.577967:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176899.577971:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311680:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176899.577974:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176899.577976:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176899.577978:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176899.577980:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176899.577984:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176899.577986:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176899.577990:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176899.577993:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176899.577997:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176899.578001:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856311680, portal 10
00000100:00000001:1.0:1786176899.578003:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176899.578004:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000040:1.0:1786176899.578012:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578019:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176899.578021:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176899.578024:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176899.578026:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856311680, offset 0
00000400:00000200:1.0:1786176899.578030:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176899.578035:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176899.578039:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176899.578043:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176899.578045:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176899.578048:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176899.578051:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176899.578054:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254606:254606:256:4294967295] 192.168.202.110@tcp LPNI seq info [254595:254595:8:4294967295]
00000400:00000200:1.0:1786176899.578058:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176899.578066:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176899.578071:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176899.578075:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176899.578079:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176899.578083:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786176899.578088:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176899.578090:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786176899.578095:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176899.578097:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176899.578101:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176899.578104:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176899.578108:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176899.578110:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176899.578112:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176899.578116:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578125:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176899.578128:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578135:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.578137:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176899.578141:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176899.578146:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176899.578149:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176899.578151:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301795338
00000100:00000001:1.0:1786176899.578154:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301799117
00000100:00000001:1.0:1786176899.578157:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301795338
00000100:00000001:1.0:1786176899.578159:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301799117
00000100:00000001:1.0:1786176899.578161:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176899.578164:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.578166:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176899.578170:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176899/real 1786176899]  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176899.578179:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176899.578180:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176899.578182:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176899.578185:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176899.578187:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176899.578189:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176899.578193:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176899.578202:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578209:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176899.578210:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.578212:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176899.578214:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176899.578218:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578225:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176899.578230:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578238:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176899.578240:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.578242:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176899.578243:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.578245:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176899.578247:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176899.578250:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176899.578254:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176899.578257:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176899.578261:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176899.578270:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311680:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176899.578274:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176899.578278:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578291:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176899.578293:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176899.578295:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176899.578297:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176899.578300:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176899.578303:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176899.578306:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176899.578308:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176899.578310:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176899.578312:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176899.578316:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176899.578318:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176899.578322:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176899.578324:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176899.578328:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176899.578331:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856311808, portal 10
00000100:00000001:1.0:1786176899.578333:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176899.578335:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:1.0:1786176899.578341:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578348:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176899.578350:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176899.578352:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176899.578355:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856311808, offset 0
00000400:00000200:1.0:1786176899.578359:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176899.578363:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176899.578367:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176899.578370:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176899.578373:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176899.578375:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176899.578378:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176899.578382:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254607:254607:256:4294967295] 192.168.202.110@tcp LPNI seq info [254596:254596:8:4294967295]
00000400:00000200:1.0:1786176899.578386:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176899.578392:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176899.578397:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176899.578401:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176899.578405:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176899.578408:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786176899.578412:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176899.578415:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786176899.578419:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176899.578421:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176899.578424:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176899.578427:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176899.578431:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176899.578432:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176899.578435:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176899.578439:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578447:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176899.578451:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578458:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.578460:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176899.578463:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176899.578466:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176899.578469:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176899.578471:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301795463
00000100:00000001:1.0:1786176899.578474:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301799117
00000100:00000001:1.0:1786176899.578476:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301795463
00000100:00000001:1.0:1786176899.578479:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301799117
00000100:00000001:1.0:1786176899.578481:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176899.578484:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.578486:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176899.578490:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176899/real 1786176899]  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176899.578498:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176899.578500:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176899.578502:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176899.578504:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176899.578506:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176899.578508:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176899.578533:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176899.578543:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578550:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176899.578552:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.578553:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176899.578555:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176899.578559:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578566:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176899.578570:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578577:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176899.578580:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.578582:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176899.578583:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.578585:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176899.578586:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176899.578589:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176899.578593:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176899.578595:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176899.578599:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176899.578608:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176899.578612:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176899.578616:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578622:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176899.578624:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176899.578627:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176899.578629:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176899.578630:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176899.578633:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176899.578652:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176899.578654:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176899.578656:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176899.578658:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176899.578662:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176899.578664:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176899.578668:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176899.578670:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176899.578674:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176899.578677:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856311936, portal 4
00000100:00000001:1.0:1786176899.578680:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176899.578682:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:1.0:1786176899.578688:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578694:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176899.578697:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176899.578700:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176899.578702:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856311936, offset 0
00000400:00000200:1.0:1786176899.578707:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176899.578711:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176899.578715:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176899.578718:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176899.578720:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176899.578723:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176899.578725:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176899.578728:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254608:254608:256:4294967295] 192.168.202.110@tcp LPNI seq info [254597:254597:8:4294967295]
00000400:00000200:1.0:1786176899.578733:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176899.578739:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176899.578744:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176899.578748:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176899.578752:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176899.578755:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786176899.578759:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176899.578761:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786176899.578766:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176899.578768:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176899.578771:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176899.578774:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176899.578777:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176899.578779:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176899.578782:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176899.578786:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578795:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176899.578798:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578805:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.578807:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176899.578811:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176899.578815:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176899.578818:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176899.578821:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301795339
00000100:00000001:1.0:1786176899.578823:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301799117
00000100:00000001:1.0:1786176899.578825:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301795339
00000100:00000001:1.0:1786176899.578827:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301799117
00000100:00000001:1.0:1786176899.578829:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176899.578832:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.578834:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176899.578838:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176899/real 1786176899]  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176899.578847:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176899.578848:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176899.578850:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176899.578852:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176899.578854:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176899.578856:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176899.578860:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176899.578869:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578875:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176899.578877:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.578879:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176899.578880:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176899.578884:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578891:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176899.578895:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578903:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176899.578906:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.578908:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176899.578909:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.578911:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176899.578913:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176899.578916:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176899.578919:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176899.578922:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176899.578926:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176899.578934:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856311936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176899.578938:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176899.578942:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.578949:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176899.578950:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176899.578953:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176899.578954:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176899.578957:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176899.578960:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856312064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176899.578963:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176899.578965:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176899.578967:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176899.578969:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176899.578973:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176899.578975:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176899.578979:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176899.578982:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176899.578985:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176899.578988:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856312064, portal 4
00000100:00000001:1.0:1786176899.578991:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176899.578992:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:1.0:1786176899.578998:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.579005:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176899.579007:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176899.579010:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176899.579012:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856312064, offset 0
00000400:00000200:1.0:1786176899.579017:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176899.579021:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176899.579025:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176899.579028:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176899.579031:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176899.579033:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176899.579035:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176899.579039:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254609:254609:256:4294967295] 192.168.202.110@tcp LPNI seq info [254598:254598:8:4294967295]
00000400:00000200:1.0:1786176899.579043:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176899.579049:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176899.579054:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176899.579058:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176899.579062:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176899.579064:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786176899.579069:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176899.579071:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786176899.579075:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176899.579077:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176899.579081:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176899.579085:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176899.579088:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176899.579090:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176899.579094:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176899.579098:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.579106:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176899.579109:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176924 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.579116:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.579118:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176899.579121:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176899.579124:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176899.579128:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176899.579130:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301795347
00000100:00000001:1.0:1786176899.579133:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301799117
00000100:00000001:1.0:1786176899.579135:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301795347
00000100:00000001:1.0:1786176899.579137:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301799117
00000100:00000001:1.0:1786176899.579139:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176899.579141:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.579143:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176899.579148:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176899/real 1786176899]  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176899.579155:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176899.579157:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176899.579159:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176899.579161:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176899.579164:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176899.579166:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176899.579170:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176899.579178:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.579191:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176899.579193:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.579195:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176899.579197:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176899.579201:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.579208:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176899.579213:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.579220:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176899.579222:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.579224:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176899.579226:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.579227:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176899.579230:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176899.579232:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176899.579235:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176899.579238:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176899.579242:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176899.579250:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856312064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176899.579254:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.579256:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176899.579260:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856311680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.579267:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176899.579268:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176899.579271:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176899.579273:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176899.579275:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176899.579277:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176899.579279:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176899.579281:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:1.0:1786176899.579284:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176899.579285:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176899.579287:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:1.0:1786176899.579289:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176899.579291:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.579293:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176899.579297:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856311808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.579304:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176899.579306:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176899.579308:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176899.579310:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176899.579312:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176899.579313:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176899.579315:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176899.579317:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176899.579319:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176899.579320:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176899.579321:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:1.0:1786176899.579324:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176899.579326:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.579327:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176899.579331:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856311936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.579337:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176899.579339:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176899.579340:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176899.579344:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176899.579345:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176899.579346:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176899.579350:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176899.579351:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786176899.579354:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176899.579355:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176899.579356:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:1.0:1786176899.579358:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176899.579360:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.579361:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176899.579365:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856312064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176899.579371:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176899.579373:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176899.579374:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176899.579377:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176899.579378:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176899.579379:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176899.579381:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176899.579383:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176899.579386:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176899.579387:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176899.579388:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:1.0:1786176899.579390:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176899.579392:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.579394:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176899.579398:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176899.579400:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176899.579402:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176899.579404:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176900.152614:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176900.152621:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176900.152623:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176900.152625:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176900.152629:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176900.152651:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176900.152653:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176900.152655:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176900.152658:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176900.152660:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176900.152661:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176900.152662:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176900.152663:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176900.152664:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176900.152665:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176900.152667:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176900.152669:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176900.152670:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176900.152732:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176900.152735:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176900.152738:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176900.600656:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176900.600665:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176900.600667:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176900.600672:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176900.600675:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176900.600678:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176900.600682:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176900.600685:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176900.600688:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176901.176680:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176901.176686:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176901.176690:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176901.176691:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176901.176693:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176901.176695:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176901.176748:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176901.176752:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176901.176754:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176901.624609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176901.624616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176901.624634:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176901.624639:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176901.624642:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176901.624645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176901.624647:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176901.624650:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176901.624652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176902.200639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176902.200644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176902.200648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176902.200649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176902.200652:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176902.200652:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176902.200709:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176902.200713:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176902.200716:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176902.648605:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176902.648637:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176902.648639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176902.648644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176902.648646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176902.648650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176902.648652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176902.648654:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176902.648656:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176903.224644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176903.224652:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176903.224658:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176903.224660:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176903.224663:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176903.224666:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176903.224669:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176903.224672:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176903.224674:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176903.224676:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176903.224678:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176903.224680:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176903.224746:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176903.224752:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176903.224756:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176903.416638:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301799117, at 4301799168
00000100:00000040:1.0:1786176903.416638:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301799117, at 4301799168
00000100:00000010:0.0:1786176903.416656:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b14e0.
00000100:00000010:1.0:1786176903.416658:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b15a0.
00000100:00000001:2.0:1786176903.672621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176903.672629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176903.672631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176903.672636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176903.672639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176903.672642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176903.672644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176903.672647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176903.672649:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176904.248644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176904.248650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176904.248655:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176904.248656:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176904.248658:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176904.248659:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176904.248734:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176904.248739:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176904.248742:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786176904.696601:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176904.696611:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176904.696612:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176904.696613:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176904.696618:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176904.696619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176904.696624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176904.696629:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176904.696629:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786176904.696635:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1786176904.696637:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7128
00000100:00000001:2.0:1786176904.696641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176904.696646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000400:00000200:3.0:1786176904.696648:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176904.696651:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1786176904.696655:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1786176904.696656:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176904.696660:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176904.696664:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176904.696667:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176904.696669:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176904.696672:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176904.696675:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176904.696676:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176904.696678:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176904.696681:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176904.696682:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176904.696684:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176904.696686:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176904.696688:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176904.696689:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176904.696690:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176904.696692:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176904.696694:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176904.696696:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176904.696698:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176904.696700:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176904.696702:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.696704:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176904.696705:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176904.696713:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d500.
00000020:00000040:3.0:1786176904.696716:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176904.696719:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176904.696720:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.696722:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176904.696725:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176904.696729:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176904.696735:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:3.0:1786176904.696740:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176904.696747:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176904.696754:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176904.696758:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176904.696786:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7c36393] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176904.696803:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176904.696805:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176904.696808:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176904.696809:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176904.696810:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176904.696812:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176904.696814:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7128
00000400:00000200:3.0:1786176904.696817:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176904.696821:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176904.696823:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176904.696825:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176904.696828:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176904.696829:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176904.696831:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176904.696833:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176904.696834:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786176904.696835:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176904.696836:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:1.0:1786176904.696837:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1786176904.696838:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176904.696842:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1786176904.696842:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176904.696846:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176904.696847:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176904.696849:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176904.696853:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176904.696855:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176904.696855:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176904.696857:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786176904.696859:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176904.696861:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176904.696864:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176904.696870:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176904.696871:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1786176904.696873:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.696874:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.696876:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786176904.696879:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:3.0:1786176904.696881:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786176904.696883:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:3.0:1786176904.696885:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176904.696887:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.696888:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176904.696892:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856312192:192.168.202.110@tcp:250:kworker.0
00000100:00000010:3.0:1786176904.696893:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ea00.
00000020:00000040:3.0:1786176904.696898:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786176904.696899:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176904.696902:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786176904.696904:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176904.696905:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176904.696908:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.696908:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176904.696911:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786176904.696913:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1786176904.696914:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176904.696919:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000010:3.0:1786176904.696919:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:1.0:1786176904.696924:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.696925:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176904.696932:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1786176904.696933:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786176904.696936:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00000001:3.0:1786176904.696943:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000010:1.0:1786176904.696944:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000001:3.0:1786176904.696945:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176904.696951:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053162887] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176904.696953:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856312192, portal 25
00000100:00000001:1.0:1786176904.696958:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176904.696960:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000100:00000001:3.0:1786176904.696963:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176904.696966:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1786176904.696968:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176904.696970:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786176904.696974:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176904.696976:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176904.696981:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176904.696983:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1786176904.696986:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7128
00000100:00000040:1.0:1786176904.696990:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786176904.696994:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176904.696995:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000400:00000200:3.0:1786176904.696998:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000200:1.0:1786176904.697000:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856312192, offset 0
00000100:00080000:3.0:1786176904.697001:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176904.697006:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176904.697008:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176904.697009:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176904.697011:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176904.697012:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:1.0:1786176904.697020:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:3.0:1786176904.697021:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786176904.697024:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176904.697026:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786176904.697028:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786176904.697029:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786176904.697033:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176904.697035:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176904.697037:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176904.697042:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176904.697045:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176904.697047:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:3.0:1786176904.697050:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:1.0:1786176904.697057:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1786176904.697057:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176904.697063:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176904.697064:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:3.0:1786176904.697068:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786176904.697070:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176904.697073:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:3.0:1786176904.697074:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786176904.697078:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254610:254610:256:4294967295] 192.168.202.110@tcp LPNI seq info [254599:254599:8:4294967295]
02000000:00000001:3.0:1786176904.697080:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.697085:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:1.0:1786176904.697086:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:3.0:1786176904.697090:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176904.697095:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176904.697098:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786176904.697100:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176904.697106:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c000.
00000800:00000200:1.0:1786176904.697107:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:3.0:1786176904.697111:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000800:00000010:1.0:1786176904.697115:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
02000000:00000001:3.0:1786176904.697115:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176904.697118:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.697120:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176904.697122:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176904.697125:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176904.697126:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176904.697131:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
02000000:00000010:3.0:1786176904.697132:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000800:00000200:1.1:1786176904.697137:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786176904.697138:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786176904.697142:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00100000:3.0:1786176904.697147:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:1.0:1786176904.697151:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176904.697155:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000100:00000001:3.0:1786176904.697161:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176904.697163:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1786176904.697163:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176904.697171:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006fe8fdb7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176904.697172:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176904.697181:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000100:00000001:3.0:1786176904.697182:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176904.697185:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00080000:3.0:1786176904.697186:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176904.697189:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1786176904.697190:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1786176904.697191:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176904.697194:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:1.0:1786176904.697197:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176904.697198:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176904.697203:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7128
00000400:00000200:3.0:1786176904.697211:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176904.697214:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000400:00000200:3.0:1786176904.697215:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176904.697219:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786176904.697221:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176904.697224:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176904.697230:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176904.697233:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176904.697235:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.697236:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176904.697237:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:3.0:1786176904.697240:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176904.697242:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786176904.697244:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176904.697245:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786176904.697246:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786176904.697250:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:1.0:1786176904.697253:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1786176904.697254:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176904.697258:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301804237
00000100:00000010:1.0:1786176904.697263:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b15a0.
00000100:00000001:1.0:1786176904.697269:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1786176904.697280:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301804237
00000100:00000010:1.0:1786176904.697284:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1540.
00000100:00000001:1.0:1786176904.697287:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:3.0:1786176904.697292:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176904.697293:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176904.697295:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176904.697299:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176904.697300:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.697302:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176904.697306:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:1.0:1786176904.697307:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00080000:3.0:1786176904.697310:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176904.697313:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176904.697315:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:1.0:1786176904.697316:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176904/real 1786176904]  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176904.697318:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176904.697322:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.697326:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176904.697328:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176904.697333:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.697334:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176904.697337:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c700.
00000020:00000040:3.0:1786176904.697338:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176904.697343:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000001:1.0:1786176904.697346:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:3.0:1786176904.697346:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176904.697350:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1786176904.697350:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:1.0:1786176904.697354:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
02000000:00000001:3.0:1786176904.697354:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176904.697357:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176904.697359:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
02000000:00000010:3.0:1786176904.697361:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000400:00000010:1.0:1786176904.697362:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:3.0:1786176904.697366:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.697367:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00100000:3.0:1786176904.697375:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1786176904.697376:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176904.697401:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1786176904.697404:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176904.697407:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.697414:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176904.697417:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176904.697418:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e56458ae] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.697419:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176904.697422:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176904.697430:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176904.697438:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.697440:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176904.697444:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1786176904.697446:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176904.697450:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786176904.697455:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786176904.697457:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786176904.697459:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176904.697460:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.697466:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176904.697466:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176904.697467:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.697471:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00080000:3.0:1786176904.697473:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7128
00000100:00000001:1.0:1786176904.697474:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176904.697478:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000400:00000200:3.0:1786176904.697482:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1786176904.697483:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176904.697488:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000400:00000200:3.0:1786176904.697491:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:1.0:1786176904.697493:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176904.697497:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176904.697504:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00100000:1.0:1786176904.697505:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856312192:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786176904.697536:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176904.697539:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176904.697542:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176904.697544:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786176904.697548:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:1.0:1786176904.697552:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856312192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786176904.697553:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176904.697556:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786176904.697559:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176904.697563:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:1.0:1786176904.697569:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000020:00000001:3.0:1786176904.697569:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786176904.697574:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
02000000:00000001:1.0:1786176904.697575:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176904.697578:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176904.697582:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000040:3.0:1786176904.697583:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000020:00000001:1.0:1786176904.697584:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000001:3.0:1786176904.697602:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176904.697605:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176904.697607:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786176904.697609:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000040:3.0:1786176904.697610:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000020:00000001:1.0:1786176904.697614:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176904.697618:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
00000100:00080000:3.0:1786176904.697618:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:1.0:1786176904.697623:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176904.697625:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000001:3.0:1786176904.697625:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786176904.697627:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
02000000:00000001:3.0:1786176904.697630:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176904.697631:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176904.697634:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786176904.697635:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176904.697636:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786176904.697639:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.697642:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176904.697644:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000008:00100000:3.0:1786176904.697644:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176904.697647:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.697648:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.697651:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.697652:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176904.697656:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176904.697658:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000010:3.0:1786176904.697658:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e300.
00000020:00000040:3.0:1786176904.697663:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176904.697667:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:1.0:1786176904.697668:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176904.697668:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.697671:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176904.697673:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176904.697676:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176904.697682:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:1.0:1786176904.697685:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176904.697686:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.697689:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176904.697694:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786176904.697695:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176904.697699:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176904.697703:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1786176904.697704:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176904.697706:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176904.697713:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856312320:192.168.202.110@tcp:38:kworker.0
00000100:00000040:3.0:1786176904.697713:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e3752a60] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176904.697720:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176904.697724:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7138)
00000100:00000001:1.0:1786176904.697727:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176904.697729:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176904.697731:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176904.697732:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176904.697736:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176904.697738:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176904.697742:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176904.697745:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176904.697749:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176904.697753:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856312320, portal 10
00000100:00000001:1.0:1786176904.697755:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176904.697757:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:1.0:1786176904.697766:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.697773:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176904.697775:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176904.697778:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176904.697781:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856312320, offset 0
00000400:00000200:1.0:1786176904.697784:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176904.697789:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176904.697793:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176904.697796:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176904.697799:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176904.697801:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176904.697804:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176904.697808:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254611:254611:256:4294967295] 192.168.202.110@tcp LPNI seq info [254600:254600:8:4294967295]
00000400:00000200:1.0:1786176904.697812:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176904.697819:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176904.697824:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176904.697828:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176904.697832:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176904.697835:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786176904.697840:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176904.697843:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1786176904.697847:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176904.697849:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176904.697852:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176904.697856:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176904.697859:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176904.697861:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176904.697864:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176904.697867:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.697873:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176904.697875:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.697880:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.697881:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176904.697883:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176904.697886:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176904.697889:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176904.697891:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301800458
00000100:00000001:1.0:1786176904.697892:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301804237
00000100:00000001:1.0:1786176904.697894:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301800458
00000100:00000001:1.0:1786176904.697896:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301804237
00000100:00000001:1.0:1786176904.697897:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176904.697899:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.697900:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176904.697903:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176904/real 1786176904]  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176904.697908:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176904.697909:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176904.697910:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176904.697912:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176904.697913:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176904.697914:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176904.697916:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176904.697922:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.697925:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176904.697926:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.697927:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176904.697928:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176904.697930:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.697934:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176904.697937:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.697941:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176904.697943:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.697944:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176904.697945:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.697946:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176904.697948:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176904.697950:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176904.697952:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176904.697953:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176904.697956:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176904.697961:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856312320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176904.697963:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176904.697965:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.697969:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176904.697970:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176904.697971:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176904.697973:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176904.697974:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176904.697976:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856312448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176904.697978:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176904.697979:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176904.697980:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176904.697982:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176904.697984:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176904.697985:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176904.697987:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176904.697989:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176904.697991:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176904.697993:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856312448, portal 10
00000100:00000001:1.0:1786176904.697994:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176904.697995:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:1.0:1786176904.697999:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698010:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176904.698013:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176904.698015:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176904.698018:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856312448, offset 0
00000400:00000200:1.0:1786176904.698022:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176904.698026:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176904.698030:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176904.698033:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176904.698035:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176904.698038:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176904.698040:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176904.698044:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254612:254612:256:4294967295] 192.168.202.110@tcp LPNI seq info [254601:254601:8:4294967295]
00000400:00000200:1.0:1786176904.698048:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176904.698055:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176904.698060:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176904.698063:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176904.698066:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176904.698068:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786176904.698070:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176904.698072:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1786176904.698075:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176904.698076:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176904.698078:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176904.698081:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176904.698083:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176904.698084:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176904.698086:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176904.698089:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698094:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176904.698096:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698105:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.698106:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176904.698108:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176904.698110:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176904.698112:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176904.698114:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301800583
00000100:00000001:1.0:1786176904.698115:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301804237
00000100:00000001:1.0:1786176904.698117:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301800583
00000100:00000001:1.0:1786176904.698119:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301804237
00000100:00000001:1.0:1786176904.698120:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176904.698121:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.698122:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176904.698125:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176904/real 1786176904]  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176904.698130:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176904.698130:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176904.698132:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176904.698133:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176904.698134:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176904.698135:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176904.698137:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176904.698142:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698146:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176904.698147:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698148:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176904.698149:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176904.698151:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698156:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176904.698158:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698162:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176904.698164:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698165:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176904.698166:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698167:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176904.698168:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176904.698169:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176904.698171:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176904.698173:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176904.698175:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176904.698180:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856312448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176904.698182:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176904.698184:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698188:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176904.698190:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176904.698192:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176904.698194:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176904.698196:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176904.698199:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856312576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176904.698202:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176904.698204:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176904.698206:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176904.698208:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176904.698211:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176904.698213:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176904.698217:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176904.698220:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176904.698223:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176904.698227:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856312576, portal 4
00000100:00000001:1.0:1786176904.698228:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176904.698230:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:1.0:1786176904.698236:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698242:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176904.698244:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176904.698247:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176904.698249:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856312576, offset 0
00000400:00000200:1.0:1786176904.698253:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176904.698258:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176904.698262:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176904.698265:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176904.698268:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176904.698270:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176904.698273:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176904.698276:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254613:254613:256:4294967295] 192.168.202.110@tcp LPNI seq info [254602:254602:8:4294967295]
00000400:00000200:1.0:1786176904.698281:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176904.698286:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176904.698289:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176904.698292:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176904.698295:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176904.698297:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786176904.698299:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176904.698301:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1786176904.698304:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176904.698305:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176904.698307:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176904.698310:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176904.698312:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176904.698313:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176904.698314:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176904.698317:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698323:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176904.698325:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698329:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.698330:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176904.698332:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176904.698335:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176904.698337:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176904.698338:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301800458
00000100:00000001:1.0:1786176904.698340:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301804237
00000100:00000001:1.0:1786176904.698341:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301800458
00000100:00000001:1.0:1786176904.698343:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301804237
00000100:00000001:1.0:1786176904.698344:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176904.698345:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.698346:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176904.698349:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176904/real 1786176904]  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176904.698356:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176904.698358:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176904.698359:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176904.698362:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176904.698363:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176904.698366:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176904.698370:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176904.698379:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698386:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176904.698387:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698389:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176904.698391:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176904.698395:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698402:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176904.698407:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698414:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176904.698416:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698418:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176904.698419:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698421:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176904.698423:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176904.698426:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176904.698430:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176904.698432:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176904.698436:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176904.698445:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856312576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176904.698449:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698451:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176904.698455:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856312320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698463:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176904.698464:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176904.698466:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176904.698469:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176904.698471:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176904.698472:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176904.698474:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176904.698476:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:1.0:1786176904.698479:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176904.698480:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176904.698481:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:1.0:1786176904.698484:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176904.698485:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698486:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176904.698489:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856312448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698494:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176904.698494:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176904.698495:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176904.698498:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176904.698499:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176904.698499:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176904.698501:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176904.698502:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176904.698503:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176904.698504:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176904.698505:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:1.0:1786176904.698506:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176904.698507:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698508:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176904.698525:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856312576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698529:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176904.698530:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176904.698531:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176904.698533:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176904.698534:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176904.698535:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176904.698536:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176904.698537:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786176904.698538:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176904.698539:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176904.698540:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:1.0:1786176904.698541:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176904.698542:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698543:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698546:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176904.698547:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.698548:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176904.698549:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176904.698551:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176904.698553:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698558:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176904.698559:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176904.698560:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176904.698562:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176904.698563:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176904.698565:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856312704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176904.698567:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176904.698568:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176904.698569:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176904.698570:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176904.698573:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176904.698574:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176904.698577:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176904.698578:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176904.698581:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176904.698583:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856312704, portal 4
00000100:00000001:1.0:1786176904.698585:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176904.698586:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:1.0:1786176904.698604:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698609:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176904.698610:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176904.698612:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176904.698614:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856312704, offset 0
00000400:00000200:1.0:1786176904.698616:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176904.698620:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176904.698622:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176904.698624:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176904.698626:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176904.698628:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176904.698629:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176904.698632:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254614:254614:256:4294967295] 192.168.202.110@tcp LPNI seq info [254603:254603:8:4294967295]
00000400:00000200:1.0:1786176904.698634:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176904.698639:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176904.698641:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176904.698643:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176904.698646:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176904.698648:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786176904.698651:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176904.698652:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1786176904.698655:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176904.698657:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176904.698659:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176904.698661:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176904.698663:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176904.698665:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176904.698667:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176904.698671:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698685:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176904.698688:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698695:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.698697:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176904.698700:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176904.698703:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176904.698707:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176904.698709:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301800467
00000100:00000001:1.0:1786176904.698712:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301804237
00000100:00000001:1.0:1786176904.698714:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301800467
00000100:00000001:1.0:1786176904.698716:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301804237
00000100:00000001:1.0:1786176904.698719:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176904.698721:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.698722:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176904.698726:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176904/real 1786176904]  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176904.698731:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176904.698732:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176904.698734:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176904.698735:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176904.698737:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176904.698738:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176904.698741:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176904.698750:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698757:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176904.698759:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698761:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176904.698763:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176904.698767:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698774:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176904.698778:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698785:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176904.698788:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698789:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176904.698790:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698792:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176904.698793:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176904.698795:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176904.698797:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176904.698799:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176904.698801:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176904.698807:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856312704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176904.698809:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698810:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176904.698812:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856312704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176929 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176904.698816:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176904.698817:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176904.698818:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176904.698820:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176904.698821:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176904.698822:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176904.698824:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176904.698825:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176904.698827:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176904.698827:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176904.698828:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:1.0:1786176904.698830:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176904.698831:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698832:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176904.698835:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176904.698836:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176904.698837:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176904.698838:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176905.272641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176905.272647:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176905.272651:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176905.272652:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176905.272655:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176905.272655:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176905.272721:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176905.272726:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176905.272730:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176905.720632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176905.720638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176905.720639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176905.720644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176905.720647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176905.720650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176905.720652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176905.720655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176905.720657:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176906.296631:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176906.296638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176906.296643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176906.296644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176906.296648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176906.296649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176906.296702:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176906.296706:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176906.296708:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176906.744644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176906.744652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176906.744654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176906.744659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176906.744663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176906.744666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176906.744670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176906.744673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176906.744676:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176907.320655:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176907.320661:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176907.320666:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176907.320667:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176907.320669:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176907.320670:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176907.320730:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176907.320735:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176907.320738:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176907.768627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176907.768641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176907.768643:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176907.768649:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176907.768652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176907.768655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176907.768657:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176907.768659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176907.768661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176908.344649:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176908.344655:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176908.344659:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176908.344660:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176908.344663:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176908.344664:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176908.344721:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176908.344725:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176908.344727:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176908.536634:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301804237, at 4301804288
00000100:00000040:1.0:1786176908.536634:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301804237, at 4301804288
00000100:00000010:1.0:1786176908.536651:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1540.
00000100:00000010:0.0:1786176908.536653:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b15a0.
00000100:00000001:2.0:1786176908.792598:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176908.792604:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176908.792605:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176908.792609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176908.792612:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176908.792615:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176908.792618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176908.792620:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176908.792622:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176909.368630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176909.368637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176909.368643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176909.368644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176909.368648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176909.368649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176909.368707:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176909.368711:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176909.368715:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176909.816616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176909.816624:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176909.816627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176909.816629:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1786176909.816634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786176909.816635:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176909.816638:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786176909.816641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1786176909.816642:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176909.816646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176909.816647:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786176909.816652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1786176909.816656:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7133
00000100:00000001:3.0:1786176909.816658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176909.816663:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786176909.816667:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176909.816673:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176909.816675:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176909.816678:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.816681:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176909.816684:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.816686:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176909.816689:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176909.816691:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176909.816693:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176909.816696:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176909.816697:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176909.816699:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176909.816702:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.816703:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176909.816704:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.816705:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176909.816707:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176909.816710:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176909.816711:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176909.816713:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176909.816716:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176909.816718:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.816720:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176909.816722:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176909.816729:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786176909.816731:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176909.816735:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176909.816736:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.816738:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176909.816741:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176909.816745:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176909.816751:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786176909.816755:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176909.816765:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176909.816777:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176909.816782:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176909.816789:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176909.816810:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176909.816814:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176909.816818:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176909.816820:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176909.816821:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176909.816825:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176909.816827:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7133
00000400:00000200:2.0:1786176909.816832:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176909.816835:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176909.816837:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176909.816839:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176909.816841:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786176909.816841:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176909.816846:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1786176909.816846:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176909.816850:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786176909.816850:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176909.816854:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176909.816855:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176909.816856:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176909.816858:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176909.816859:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176909.816863:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:1.0:1786176909.816864:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176909.816867:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.816870:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176909.816873:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.816877:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176909.816881:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786176909.816887:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1786176909.816888:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176909.816889:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.816892:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176909.816894:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786176909.816895:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786176909.816897:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176909.816902:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176909.816904:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176909.816906:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.816909:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00100000:1.0:1786176909.816912:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856312832:192.168.202.110@tcp:250:kworker.0
00000008:00100000:2.0:1786176909.816912:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1786176909.816919:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1786176909.816919:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.816921:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176909.816924:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:2.0:1786176909.816925:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
02000000:00000001:1.0:1786176909.816927:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176909.816930:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:2.0:1786176909.816930:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176909.816935:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176909.816936:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176909.816938:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
00000100:00000001:2.0:1786176909.816938:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176909.816941:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786176909.816942:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176909.816944:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176909.816949:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:2.0:1786176909.816950:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000400:00000010:1.0:1786176909.816954:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00000001:2.0:1786176909.816955:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176909.816962:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00100000:2.0:1786176909.816964:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1786176909.816969:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856312832, portal 25
00000100:00000001:1.0:1786176909.816974:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176909.816976:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187904 : -114548255363712 : ffff97d1a7eded80)
00000100:00000001:2.0:1786176909.816984:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1786176909.816985:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176909.816988:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176909.816998:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.816999:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176909.817007:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176909.817008:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176909.817012:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00080000:2.0:1786176909.817012:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176909.817018:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000200:1.0:1786176909.817019:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856312832, offset 0
00000100:00000001:2.0:1786176909.817020:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176909.817022:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176909.817026:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176909.817029:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7133
00000400:00000200:1.0:1786176909.817034:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786176909.817035:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176909.817040:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176909.817042:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176909.817044:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.817046:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176909.817048:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786176909.817050:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.817053:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176909.817055:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176909.817056:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000200:1.0:1786176909.817059:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:2.0:1786176909.817060:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176909.817065:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:1.0:1786176909.817066:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:2.0:1786176909.817072:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786176909.817073:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176909.817078:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:2.0:1786176909.817078:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786176909.817081:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176909.817088:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254615:254615:256:4294967295] 192.168.202.110@tcp LPNI seq info [254604:254604:8:4294967295]
00000100:00000040:2.0:1786176909.817090:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176909.817094:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176909.817096:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176909.817102:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176909.817109:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786176909.817109:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176909.817116:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000800:00000200:1.0:1786176909.817117:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176909.817121:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000010:1.0:1786176909.817123:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000100:00000001:2.0:1786176909.817125:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176909.817128:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176909.817130:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176909.817132:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.817133:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176909.817135:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176909.817137:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.817142:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176909.817145:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:2.0:1786176909.817145:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000020:00000040:2.0:1786176909.817149:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176909.817153:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:1.0:1786176909.817154:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
02000000:00000001:2.0:1786176909.817154:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.817157:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176909.817160:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176909.817162:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786176909.817167:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:2.0:1786176909.817168:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786176909.817174:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786176909.817174:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00100000:2.0:1786176909.817183:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:1.0:1786176909.817184:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176909.817189:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000100:00000001:2.0:1786176909.817193:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176909.817196:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176909.817198:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1786176909.817200:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176909.817209:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1786176909.817211:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176909.817214:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786176909.817216:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000100:00080000:2.0:1786176909.817217:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176909.817221:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:1.0:1786176909.817223:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00080000:2.0:1786176909.817223:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176909.817229:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176909.817232:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:2.0:1786176909.817232:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7133
00000400:00000200:2.0:1786176909.817238:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:1.0:1786176909.817242:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786176909.817244:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176909.817249:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176909.817255:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.817259:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176909.817263:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176909.817265:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786176909.817265:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176909.817270:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176909.817271:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176909.817273:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000040:1.0:1786176909.817274:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176909.817275:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176909.817280:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176909.817283:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176909.817286:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.817287:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176909.817288:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.817289:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176909.817294:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176909.817299:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176909.817304:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.817306:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176909.817307:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176909.817310:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176909.817311:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176909.817320:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176909.817321:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.817325:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176909.817326:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:1.0:1786176909.817328:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:2.0:1786176909.817329:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.817333:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.817334:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301809357
00000100:00000010:2.0:1786176909.817335:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786176909.817341:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176909.817342:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176909.817344:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786176909.817345:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1540.
00000100:00000001:2.0:1786176909.817345:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176909.817348:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176909.817351:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.817355:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000010:2.0:1786176909.817355:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786176909.817361:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.817363:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301809357
00000100:00100000:2.0:1786176909.817365:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:1.0:1786176909.817375:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1360.
00000100:00000001:2.0:1786176909.817375:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176909.817379:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.817381:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:2.0:1786176909.817385:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.817386:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176909.817392:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.817396:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1786176909.817401:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176909.817404:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176909/real 1786176909]  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176909.817404:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176909.817413:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176909.817416:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176909.817419:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000001:1.0:1786176909.817421:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786176909.817423:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176909.817425:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7133
00000400:00000001:1.0:1786176909.817429:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000200:2.0:1786176909.817431:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176909.817433:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:2.0:1786176909.817436:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1786176909.817438:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000100:00080000:2.0:1786176909.817439:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:1.0:1786176909.817442:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000040:2.0:1786176909.817444:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176909.817447:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1786176909.817448:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1786176909.817453:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176909.817454:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.817459:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176909.817464:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176909.817467:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176909.817470:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000200:1.0:1786176909.817471:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176909.817472:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176909.817477:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786176909.817482:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176909.817485:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176909.817488:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786176909.817488:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.817492:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176909.817493:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176909.817496:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786176909.817497:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176909.817500:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00100000:1.0:1786176909.817502:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176909.817503:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:1.0:1786176909.817533:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786176909.817534:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176909.817553:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176909.817555:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176909.817558:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176909.817559:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.817560:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176909.817561:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176909.817563:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.817564:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176909.817568:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786176909.817570:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176909.817572:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176909.817573:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176909.817574:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176909.817575:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176909.817576:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176909.817579:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786176909.817582:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176909.817586:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176909.817591:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176909.817593:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176909.817595:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176909.817600:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176909.817604:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7143)
00000100:00000040:1.0:1786176909.817676:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.817686:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176909.817689:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.817691:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176909.817692:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.817694:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176909.817696:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176909.817699:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176909.817704:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176909.817707:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176909.817713:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176909.817723:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856312832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176909.817729:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.817730:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176909.817733:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eded80 x1872934856312832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.817737:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176909.817739:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176909.817741:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176909.817744:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176909.817745:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176909.817747:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176909.817748:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176909.817750:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:1.0:1786176909.817753:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176909.817754:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176909.817756:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eded80.
00000100:00000001:1.0:1786176909.817758:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176909.817759:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.817761:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.817766:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176909.817767:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.817768:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176909.817770:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176909.817771:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176909.817773:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.817778:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176909.817779:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176909.817781:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176909.817782:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176909.817784:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176909.817786:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856312960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176909.817788:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176909.817790:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176909.817791:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176909.817792:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176909.817795:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176909.817796:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176909.817799:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176909.817801:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176909.817804:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176909.817806:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856312960, portal 10
00000100:00000001:1.0:1786176909.817807:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176909.817808:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454180736 : -114548255370880 : ffff97d1a7edd180)
00000100:00000040:1.0:1786176909.817813:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.817818:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176909.817820:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176909.817821:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176909.817823:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856312960, offset 0
00000400:00000200:1.0:1786176909.817825:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176909.817828:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176909.817831:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176909.817834:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176909.817835:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176909.817837:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176909.817839:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176909.817841:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254616:254616:256:4294967295] 192.168.202.110@tcp LPNI seq info [254605:254605:8:4294967295]
00000400:00000200:1.0:1786176909.817844:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176909.817848:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176909.817851:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176909.817853:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176909.817856:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176909.817859:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786176909.817862:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176909.817864:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1786176909.817866:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176909.817868:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176909.817870:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176909.817872:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176909.817874:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176909.817875:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176909.817877:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176909.817880:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.817885:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176909.817887:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.817892:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.817893:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176909.817895:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176909.817898:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176909.817900:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176909.817901:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301805578
00000100:00000001:1.0:1786176909.817904:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301809357
00000100:00000001:1.0:1786176909.817905:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301805578
00000100:00000001:1.0:1786176909.817907:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301809357
00000100:00000001:1.0:1786176909.817908:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176909.817910:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.817911:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176909.817913:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176909/real 1786176909]  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176909.817918:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176909.817919:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176909.817920:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176909.817921:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176909.817922:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176909.817924:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176909.817926:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176909.817931:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.817935:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176909.817936:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.817937:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176909.817938:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176909.817940:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.817944:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176909.817946:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.817951:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176909.817953:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.817954:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176909.817955:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.817956:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176909.817958:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176909.817960:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176909.817962:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176909.817963:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176909.817965:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176909.817970:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856312960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176909.817972:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176909.817974:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.817978:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176909.817980:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176909.817981:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176909.817982:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176909.817983:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176909.817985:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176909.817987:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176909.817988:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176909.817989:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176909.817990:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176909.817992:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176909.817993:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176909.817995:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176909.817996:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176909.817999:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176909.818002:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856313088, portal 10
00000100:00000001:1.0:1786176909.818004:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176909.818005:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178048 : -114548255373568 : ffff97d1a7edc700)
00000100:00000040:1.0:1786176909.818011:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818018:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176909.818020:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176909.818023:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176909.818026:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856313088, offset 0
00000400:00000200:1.0:1786176909.818030:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176909.818034:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176909.818038:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176909.818041:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176909.818044:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176909.818046:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176909.818049:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176909.818052:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254617:254617:256:4294967295] 192.168.202.110@tcp LPNI seq info [254606:254606:8:4294967295]
00000400:00000200:1.0:1786176909.818056:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176909.818061:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176909.818064:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176909.818067:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176909.818069:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176909.818071:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786176909.818074:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176909.818076:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1786176909.818079:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176909.818080:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176909.818083:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176909.818085:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176909.818087:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176909.818088:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176909.818090:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176909.818093:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818102:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176909.818104:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818108:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.818109:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176909.818111:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176909.818114:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176909.818116:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176909.818117:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301805703
00000100:00000001:1.0:1786176909.818119:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301809357
00000100:00000001:1.0:1786176909.818120:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301805703
00000100:00000001:1.0:1786176909.818122:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301809357
00000100:00000001:1.0:1786176909.818123:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176909.818124:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.818125:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176909.818128:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176909/real 1786176909]  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176909.818132:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176909.818133:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176909.818134:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176909.818135:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176909.818137:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176909.818138:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176909.818140:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176909.818145:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818149:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176909.818150:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818151:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176909.818152:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176909.818154:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818159:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176909.818161:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818165:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176909.818166:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818167:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176909.818168:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818169:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176909.818170:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176909.818172:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176909.818174:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176909.818175:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176909.818177:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176909.818182:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176909.818184:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176909.818186:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818191:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176909.818192:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176909.818193:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176909.818194:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176909.818196:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176909.818198:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176909.818200:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176909.818201:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176909.818202:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176909.818204:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176909.818206:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176909.818207:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176909.818209:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176909.818211:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176909.818213:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176909.818215:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856313216, portal 4
00000100:00000001:1.0:1786176909.818216:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176909.818217:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454181632 : -114548255369984 : ffff97d1a7edd500)
00000100:00000040:1.0:1786176909.818221:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818225:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176909.818226:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176909.818228:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176909.818229:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856313216, offset 0
00000400:00000200:1.0:1786176909.818232:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176909.818234:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176909.818236:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176909.818238:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176909.818240:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176909.818241:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176909.818243:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176909.818245:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254618:254618:256:4294967295] 192.168.202.110@tcp LPNI seq info [254607:254607:8:4294967295]
00000400:00000200:1.0:1786176909.818248:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176909.818252:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176909.818254:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176909.818256:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176909.818259:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176909.818261:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786176909.818263:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176909.818265:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1786176909.818267:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176909.818269:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176909.818271:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176909.818273:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176909.818275:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176909.818276:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176909.818277:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176909.818279:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818284:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176909.818286:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818290:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.818292:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176909.818293:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176909.818295:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176909.818297:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176909.818298:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301805578
00000100:00000001:1.0:1786176909.818300:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301809357
00000100:00000001:1.0:1786176909.818302:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301805578
00000100:00000001:1.0:1786176909.818303:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301809357
00000100:00000001:1.0:1786176909.818304:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176909.818306:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.818307:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176909.818309:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176909/real 1786176909]  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176909.818313:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176909.818314:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176909.818315:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176909.818317:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176909.818318:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176909.818319:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176909.818321:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176909.818326:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818330:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176909.818331:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818332:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176909.818333:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176909.818335:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818339:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176909.818341:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818345:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176909.818346:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818347:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176909.818348:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818349:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176909.818350:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176909.818351:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176909.818353:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176909.818354:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176909.818357:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176909.818366:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176909.818369:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176909.818371:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818375:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176909.818376:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176909.818377:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176909.818378:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176909.818379:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176909.818381:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313344:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176909.818383:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176909.818384:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176909.818385:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176909.818386:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176909.818387:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176909.818388:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176909.818390:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176909.818391:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176909.818393:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176909.818395:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856313344, portal 4
00000100:00000001:1.0:1786176909.818396:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176909.818397:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:1.0:1786176909.818401:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818405:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176909.818406:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176909.818408:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176909.818409:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856313344, offset 0
00000400:00000200:1.0:1786176909.818411:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176909.818414:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176909.818416:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176909.818418:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176909.818420:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176909.818421:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176909.818423:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176909.818425:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254619:254619:256:4294967295] 192.168.202.110@tcp LPNI seq info [254608:254608:8:4294967295]
00000400:00000200:1.0:1786176909.818427:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176909.818431:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176909.818434:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176909.818435:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176909.818438:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176909.818439:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786176909.818441:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176909.818443:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1786176909.818446:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176909.818447:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176909.818448:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176909.818451:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176909.818452:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176909.818453:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176909.818455:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176909.818457:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818462:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176909.818464:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818468:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.818469:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176909.818470:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176909.818472:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176909.818474:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176909.818475:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301805586
00000100:00000001:1.0:1786176909.818477:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301809357
00000100:00000001:1.0:1786176909.818478:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301805586
00000100:00000001:1.0:1786176909.818480:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301809357
00000100:00000001:1.0:1786176909.818481:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176909.818482:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.818483:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176909.818485:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176909/real 1786176909]  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176909.818490:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176909.818490:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176909.818491:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176909.818492:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176909.818493:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176909.818494:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176909.818496:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176909.818502:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818506:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176909.818507:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818508:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176909.818508:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176909.818527:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818532:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176909.818534:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818562:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176909.818565:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818566:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176909.818567:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818571:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176909.818572:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176909.818574:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176909.818576:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176909.818578:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176909.818581:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176909.818586:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313344:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176909.818589:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818590:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176909.818592:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd180 x1872934856312960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818597:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176909.818598:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176909.818599:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176909.818601:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176909.818602:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176909.818603:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176909.818604:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176909.818606:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:1.0:1786176909.818608:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176909.818609:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176909.818610:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd180.
00000100:00000001:1.0:1786176909.818613:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176909.818614:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818616:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176909.818619:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc700 x1872934856313088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818632:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176909.818634:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176909.818635:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176909.818638:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176909.818639:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176909.818640:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176909.818643:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176909.818644:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:1.0:1786176909.818646:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176909.818647:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176909.818649:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc700.
00000100:00000001:1.0:1786176909.818651:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176909.818653:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818655:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176909.818658:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd500 x1872934856313216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818664:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176909.818665:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176909.818666:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176909.818669:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176909.818670:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176909.818671:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176909.818672:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176909.818673:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:1.0:1786176909.818675:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176909.818675:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176909.818676:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd500.
00000100:00000001:1.0:1786176909.818678:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176909.818678:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818679:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176909.818682:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856313344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176909.818689:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176909.818690:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176909.818692:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00000.
02000000:00000001:1.0:1786176909.818694:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176909.818695:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176909.818697:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176909.818699:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176909.818700:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:1.0:1786176909.818704:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176909.818705:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176909.818707:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:1.0:1786176909.818709:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176909.818711:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818713:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176909.818717:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176909.818718:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176909.818720:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176909.818722:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176910.392629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176910.392640:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176910.392642:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176910.392647:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176910.392653:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176910.392656:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176910.392658:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176910.392660:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176910.392665:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176910.392667:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176910.392668:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176910.392670:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176910.392672:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176910.392673:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176910.392675:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176910.392677:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176910.392680:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176910.392682:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176910.392743:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176910.392747:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176910.392750:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176910.840636:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176910.840645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176910.840648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176910.840655:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176910.840657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176910.840661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176910.840663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176910.840665:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176910.840668:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176911.416630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176911.416636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176911.416641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176911.416643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176911.416646:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176911.416648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176911.416718:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176911.416722:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176911.416724:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176911.864622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176911.864630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176911.864632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176911.864639:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176911.864642:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176911.864646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176911.864650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176911.864653:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176911.864657:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176912.440601:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176912.440608:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176912.440612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176912.440613:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176912.440616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176912.440617:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176912.440662:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176912.440666:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176912.440669:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176912.888614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176912.888621:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176912.888623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176912.888628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176912.888631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176912.888634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176912.888637:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176912.888639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176912.888641:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176913.464619:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176913.464627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176913.464633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176913.464634:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176913.464637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176913.464640:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176913.464643:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176913.464646:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176913.464649:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176913.464651:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176913.464653:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176913.464655:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176913.464712:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176913.464717:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176913.464721:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786176913.656634:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301809357, at 4301809408
00000100:00000010:1.0:1786176913.656649:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1360.
00000100:00000040:0.0:1786176913.656739:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301809357, at 4301809408
00000100:00000010:0.0:1786176913.656745:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1540.
00000800:00000200:3.0:1786176913.784588:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:3.0:1786176913.784596:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:3.0:1786176913.784600:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1786176913.784748:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:1.0:1786176913.784755:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1786176913.784763:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1786176913.784770:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786176913.784774:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176913.912587:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176913.912594:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176913.912596:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176913.912600:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176913.912603:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176913.912606:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176913.912609:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176913.912611:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176913.912613:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176914.488642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176914.488649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176914.488656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176914.488658:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176914.488661:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176914.488663:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176914.488736:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176914.488741:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176914.488745:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176914.936634:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176914.936644:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176914.936646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176914.936654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176914.936659:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176914.936663:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176914.936666:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176914.936669:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176914.936672:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176914.936684:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176914.936699:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176914.936703:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176914.936708:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176914.936713:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176914.936722:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7138
00000400:00000200:2.0:1786176914.936733:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176914.936741:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176914.936745:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176914.936749:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.936752:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176914.936755:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.936758:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176914.936763:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176914.936765:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176914.936768:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176914.936787:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176914.936789:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176914.936792:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176914.936794:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.936796:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176914.936797:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.936799:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176914.936801:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176914.936804:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176914.936806:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176914.936808:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176914.936811:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176914.936814:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.936815:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176914.936817:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176914.936826:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786176914.936829:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176914.936832:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176914.936834:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.936836:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176914.936839:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176914.936842:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176914.936848:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786176914.936854:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176914.936861:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176914.936869:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176914.936873:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176914.936876:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176914.936895:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176914.936898:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176914.936900:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176914.936901:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176914.936903:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176914.936905:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176914.936907:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7138
00000400:00000200:2.0:1786176914.936909:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176914.936911:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176914.936913:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176914.936915:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.936917:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176914.936918:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.936920:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176914.936922:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176914.936922:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176914.936924:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176914.936925:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176914.936927:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176914.936928:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176914.936930:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.936932:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176914.936933:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.936935:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176914.936936:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176914.936938:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176914.936940:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786176914.936940:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176914.936944:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176914.936946:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176914.936949:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:2.0:1786176914.936949:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176914.936954:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.936956:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:1.0:1786176914.936959:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1786176914.936960:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176914.936966:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.936970:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.936974:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176914.936977:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176914.936977:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
02000000:00000001:1.0:1786176914.936982:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:2.0:1786176914.936983:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786176914.936985:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176914.936989:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176914.936990:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:1.0:1786176914.936993:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856313472:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1786176914.936994:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.936999:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1786176914.937000:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176914.937003:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786176914.937004:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176914.937007:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176914.937007:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176914.937013:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786176914.937016:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
02000000:00000010:1.0:1786176914.937019:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176914.937022:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.937023:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176914.937029:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176914.937034:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00100000:2.0:1786176914.937034:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786176914.937041:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000001:2.0:1786176914.937048:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1786176914.937050:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856313472, portal 25
00000100:00000001:2.0:1786176914.937051:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.937055:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176914.937058:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000040:2.0:1786176914.937060:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786176914.937066:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176914.937078:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.937079:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1786176914.937082:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1786176914.937083:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176914.937088:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:1.0:1786176914.937089:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000001:2.0:1786176914.937091:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:1.0:1786176914.937093:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856313472, offset 0
00000100:00080000:2.0:1786176914.937094:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176914.937099:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786176914.937103:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176914.937103:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7138
00000400:00000200:2.0:1786176914.937109:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176914.937112:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1786176914.937115:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786176914.937116:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176914.937121:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.937123:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176914.937126:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1786176914.937128:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.937131:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176914.937133:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:2.0:1786176914.937134:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176914.937136:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176914.937138:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786176914.937140:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:2.0:1786176914.937141:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:1.0:1786176914.937144:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:2.0:1786176914.937144:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176914.937147:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786176914.937150:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:2.0:1786176914.937153:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.937157:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176914.937158:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254620:254620:256:4294967295] 192.168.202.110@tcp LPNI seq info [254609:254609:8:4294967295]
00000100:00000040:2.0:1786176914.937162:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176914.937166:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176914.937167:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176914.937171:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176914.937173:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176914.937176:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176914.937179:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176914.937183:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786176914.937184:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1786176914.937186:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.937190:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176914.937193:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176914.937195:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.937198:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176914.937200:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000010:2.0:1786176914.937201:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786176914.937206:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176914.937207:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176914.937208:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786176914.937210:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000100:00000001:2.0:1786176914.937210:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176914.937212:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176914.937216:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176914.937220:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000800:00000200:1.0:1786176914.937221:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176914.937224:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176914.937228:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00100000:2.0:1786176914.937231:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.1:1786176914.937239:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176914.937241:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176914.937244:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786176914.937247:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1786176914.937249:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:1.0:1786176914.937259:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176914.937262:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786176914.937264:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000100:00080000:2.0:1786176914.937265:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176914.937270:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176914.937271:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176914.937273:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786176914.937275:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1786176914.937276:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176914.937280:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7138
00000400:00000200:2.0:1786176914.937285:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176914.937287:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786176914.937291:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176914.937293:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786176914.937294:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000100:00000040:2.0:1786176914.937295:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:1.0:1786176914.937298:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:2.0:1786176914.937299:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176914.937303:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.937305:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176914.937306:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:2.0:1786176914.937307:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176914.937309:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000200:1.0:1786176914.937312:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786176914.937313:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176914.937317:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176914.937323:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176914.937326:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000001:2.0:1786176914.937328:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:1.0:1786176914.937330:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176914.937337:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176914.937340:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.937342:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176914.937343:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:2.0:1786176914.937349:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176914.937351:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.937353:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176914.937357:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786176914.937359:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:1.0:1786176914.937361:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301814477
00000100:00080000:2.0:1786176914.937367:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000010:1.0:1786176914.937368:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1360.
00000100:00000001:2.0:1786176914.937374:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176914.937376:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176914.937377:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:2.0:1786176914.937380:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176914.937385:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301814477
02000000:00000001:2.0:1786176914.937385:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.937391:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000010:1.0:1786176914.937392:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b19c0.
00000008:00100000:2.0:1786176914.937393:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786176914.937395:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000008:00000001:2.0:1786176914.937397:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.937400:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.937401:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176914.937405:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176914.937406:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000100:00000001:1.0:1786176914.937408:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000020:00000040:2.0:1786176914.937410:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00100000:1.0:1786176914.937415:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176914/real 1786176914]  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176914.937416:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176914.937419:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.937423:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176914.937425:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176914.937428:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176914.937433:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786176914.937438:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176914.937440:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176914.937443:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00100000:2.0:1786176914.937445:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786176914.937446:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176914.937451:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176914.937454:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176914.937458:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1786176914.937460:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176914.937463:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176914.937465:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176914.937469:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176914.937486:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176914.937499:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176914.937505:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176914.937551:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176914.937554:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176914.937556:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176914.937559:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176914.937565:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7138
00000100:00000001:1.0:1786176914.937568:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176914.937571:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.937576:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000400:00000200:2.0:1786176914.937576:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176914.937580:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1786176914.937585:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00100000:1.0:1786176914.937588:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176914.937590:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176914.937599:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176914.937602:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176914.937605:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786176914.937610:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176914.937613:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176914.937618:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176914.937625:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176914.937628:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:1.0:1786176914.937629:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:2.0:1786176914.937629:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176914.937633:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176914.937652:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176914.937653:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1786176914.937654:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176914.937658:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:2.0:1786176914.937659:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176914.937663:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176914.937663:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176914.937666:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176914.937669:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786176914.937669:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176914.937673:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:1.0:1786176914.937674:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176914.937680:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786176914.937683:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00080000:1.0:1786176914.937686:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786176914.937687:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.937691:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:2.0:1786176914.937693:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176914.937697:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1786176914.937698:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176914.937700:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.937705:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176914.937707:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176914.937712:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.937715:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176914.937716:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856313472:192.168.202.110@tcp:250:kworker.0
00000100:00000010:2.0:1786176914.937719:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000100:00000001:1.0:1786176914.937721:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786176914.937723:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176914.937725:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1786176914.937727:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:1.0:1786176914.937730:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856313472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176914.937730:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176914.937734:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176914.937736:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176914.937739:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.937742:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176914.937745:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786176914.937745:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
02000000:00000010:1.0:1786176914.937749:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
00000100:00000001:2.0:1786176914.937750:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176914.937755:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00100000:2.0:1786176914.937757:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000020:00000001:1.0:1786176914.937758:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176914.937762:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176914.937767:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00000001:2.0:1786176914.937770:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000010:1.0:1786176914.937771:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786176914.937775:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176914.937777:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176914.937779:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000040:2.0:1786176914.937780:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1786176914.937781:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:1.0:1786176914.937787:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176914.937790:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176914.937792:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176914.937797:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7148)
00000100:00000001:1.0:1786176914.937800:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.937808:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176914.937813:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.937815:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176914.937816:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176914.937817:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176914.937821:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.937827:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176914.937828:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176914.937830:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176914.937831:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176914.937832:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176914.937834:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313600:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176914.937837:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176914.937838:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176914.937839:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176914.937841:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176914.937843:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176914.937844:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176914.937847:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176914.937849:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176914.937852:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176914.937855:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856313600, portal 10
00000100:00000001:1.0:1786176914.937856:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176914.937857:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:1.0:1786176914.937862:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.937867:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176914.937868:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176914.937870:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176914.937871:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856313600, offset 0
00000400:00000200:1.0:1786176914.937874:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176914.937877:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176914.937879:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176914.937881:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176914.937883:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176914.937885:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176914.937887:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176914.937889:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254621:254621:256:4294967295] 192.168.202.110@tcp LPNI seq info [254610:254610:8:4294967295]
00000400:00000200:1.0:1786176914.937892:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176914.937896:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176914.937899:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176914.937901:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176914.937904:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176914.937906:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786176914.937909:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176914.937911:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786176914.937914:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176914.937915:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176914.937918:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176914.937920:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176914.937922:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176914.937923:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176914.937925:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176914.937927:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.937932:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176914.937934:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.937939:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.937940:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176914.937942:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176914.937945:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176914.937947:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176914.937948:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301810698
00000100:00000001:1.0:1786176914.937951:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301814477
00000100:00000001:1.0:1786176914.937952:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301810698
00000100:00000001:1.0:1786176914.937954:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301814477
00000100:00000001:1.0:1786176914.937955:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176914.937956:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.937957:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176914.937960:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176914/real 1786176914]  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176914.937964:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176914.937965:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176914.937967:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176914.937968:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176914.937969:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176914.937971:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176914.937973:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176914.937978:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.937982:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176914.937983:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.937984:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176914.937984:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176914.937987:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.937991:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176914.937993:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.937998:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176914.938000:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938001:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176914.938001:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938003:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176914.938004:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176914.938005:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176914.938007:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176914.938009:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176914.938011:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176914.938016:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313600:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176914.938018:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176914.938020:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938025:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176914.938026:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176914.938027:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176914.938028:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176914.938029:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176914.938031:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176914.938033:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176914.938034:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176914.938034:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176914.938035:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176914.938037:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176914.938039:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176914.938041:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176914.938042:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176914.938044:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176914.938046:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856313728, portal 10
00000100:00000001:1.0:1786176914.938047:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176914.938048:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:1.0:1786176914.938052:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938056:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176914.938057:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176914.938059:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176914.938060:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856313728, offset 0
00000400:00000200:1.0:1786176914.938063:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176914.938065:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176914.938067:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176914.938069:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176914.938070:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176914.938072:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176914.938073:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176914.938076:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254622:254622:256:4294967295] 192.168.202.110@tcp LPNI seq info [254611:254611:8:4294967295]
00000400:00000200:1.0:1786176914.938078:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176914.938082:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176914.938085:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176914.938087:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176914.938089:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176914.938090:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786176914.938093:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176914.938094:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786176914.938097:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176914.938099:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176914.938100:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176914.938103:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176914.938104:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176914.938106:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176914.938107:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176914.938110:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938114:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176914.938116:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938121:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.938122:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176914.938123:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176914.938125:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176914.938127:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176914.938128:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301810823
00000100:00000001:1.0:1786176914.938129:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301814477
00000100:00000001:1.0:1786176914.938131:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301810823
00000100:00000001:1.0:1786176914.938132:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301814477
00000100:00000001:1.0:1786176914.938134:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176914.938135:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.938136:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176914.938139:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176914/real 1786176914]  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176914.938143:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176914.938144:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176914.938145:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176914.938146:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176914.938147:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176914.938148:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176914.938150:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176914.938155:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938159:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176914.938160:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938161:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176914.938162:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176914.938164:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938168:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176914.938170:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938177:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176914.938179:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938180:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176914.938181:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938184:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176914.938186:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176914.938188:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176914.938191:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176914.938194:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176914.938198:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176914.938207:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176914.938211:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176914.938215:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938223:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176914.938224:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176914.938227:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176914.938228:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176914.938230:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176914.938232:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176914.938234:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176914.938236:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176914.938238:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176914.938240:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176914.938243:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176914.938245:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176914.938249:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176914.938252:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176914.938256:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176914.938259:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856313856, portal 4
00000100:00000001:1.0:1786176914.938261:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176914.938263:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:1.0:1786176914.938269:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938277:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176914.938280:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176914.938283:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176914.938285:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856313856, offset 0
00000400:00000200:1.0:1786176914.938288:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176914.938291:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176914.938294:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176914.938296:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176914.938297:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176914.938299:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176914.938301:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176914.938303:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254623:254623:256:4294967295] 192.168.202.110@tcp LPNI seq info [254612:254612:8:4294967295]
00000400:00000200:1.0:1786176914.938305:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176914.938310:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176914.938313:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176914.938315:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176914.938317:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176914.938319:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786176914.938322:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176914.938324:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786176914.938326:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176914.938328:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176914.938330:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176914.938332:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176914.938334:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176914.938335:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176914.938337:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176914.938339:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938345:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176914.938347:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938356:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.938357:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176914.938359:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176914.938361:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176914.938363:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176914.938364:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301810698
00000100:00000001:1.0:1786176914.938366:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301814477
00000100:00000001:1.0:1786176914.938368:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301810698
00000100:00000001:1.0:1786176914.938369:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301814477
00000100:00000001:1.0:1786176914.938370:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176914.938371:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.938373:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176914.938375:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176914/real 1786176914]  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176914.938380:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176914.938381:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176914.938382:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176914.938383:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176914.938384:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176914.938386:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176914.938388:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176914.938393:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938397:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176914.938398:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938399:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176914.938399:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176914.938402:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938406:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176914.938409:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938413:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176914.938415:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938415:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176914.938416:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938417:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176914.938418:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176914.938420:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176914.938422:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176914.938423:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176914.938425:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176914.938430:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176914.938432:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176914.938434:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938438:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176914.938439:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176914.938440:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176914.938441:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176914.938443:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176914.938444:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176914.938446:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176914.938447:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176914.938449:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176914.938450:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176914.938452:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176914.938453:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176914.938455:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176914.938457:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176914.938459:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176914.938460:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856313984, portal 4
00000100:00000001:1.0:1786176914.938462:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176914.938463:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00000040:1.0:1786176914.938466:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938471:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176914.938472:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176914.938473:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176914.938475:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856313984, offset 0
00000400:00000200:1.0:1786176914.938477:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176914.938479:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176914.938481:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176914.938482:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176914.938484:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176914.938485:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176914.938487:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176914.938506:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254624:254624:256:4294967295] 192.168.202.110@tcp LPNI seq info [254613:254613:8:4294967295]
00000400:00000200:1.0:1786176914.938524:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176914.938528:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176914.938532:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176914.938534:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176914.938537:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176914.938539:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786176914.938542:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176914.938544:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786176914.938546:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176914.938548:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176914.938550:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176914.938552:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176914.938554:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176914.938555:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176914.938557:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176914.938560:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938565:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176914.938567:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938572:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.938573:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176914.938575:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176914.938577:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176914.938579:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176914.938580:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301810707
00000100:00000001:1.0:1786176914.938582:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301814477
00000100:00000001:1.0:1786176914.938583:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301810707
00000100:00000001:1.0:1786176914.938584:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301814477
00000100:00000001:1.0:1786176914.938586:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176914.938587:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.938588:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176914.938590:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176914/real 1786176914]  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176914.938595:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176914.938596:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176914.938597:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176914.938599:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176914.938600:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176914.938602:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176914.938603:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176914.938609:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938613:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176914.938614:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938615:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176914.938616:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176914.938618:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938624:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176914.938628:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938634:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176914.938637:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938639:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176914.938640:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938642:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176914.938644:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176914.938646:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176914.938650:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176914.938652:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176914.938656:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176914.938666:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856313984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176914.938669:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938672:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176914.938675:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856313600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938679:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176914.938680:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176914.938682:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176914.938684:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176914.938685:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176914.938686:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176914.938688:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176914.938689:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:1.0:1786176914.938691:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176914.938692:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176914.938693:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:1.0:1786176914.938695:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176914.938696:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938697:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176914.938699:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856313728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938703:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176914.938704:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176914.938705:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176914.938706:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176914.938707:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176914.938708:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176914.938709:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176914.938710:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:1.0:1786176914.938712:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176914.938713:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176914.938713:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:1.0:1786176914.938715:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176914.938716:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938717:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176914.938718:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856313856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938722:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176914.938723:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176914.938724:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176914.938725:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176914.938726:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176914.938727:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176914.938728:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176914.938729:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:1.0:1786176914.938730:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176914.938731:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176914.938732:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:1.0:1786176914.938733:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176914.938734:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938735:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176914.938736:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856313984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176939 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176914.938740:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176914.938741:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176914.938742:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176914.938743:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176914.938743:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176914.938744:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176914.938745:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176914.938746:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:1.0:1786176914.938747:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176914.938748:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176914.938749:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:1.0:1786176914.938750:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176914.938750:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938751:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176914.938754:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176914.938755:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176914.938756:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176914.938757:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176915.512604:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176915.512614:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176915.512621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176915.512622:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176915.512628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176915.512629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176915.512703:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176915.512709:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176915.512713:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176915.960630:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176915.960636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176915.960638:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176915.960642:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176915.960645:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176915.960648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176915.960650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176915.960652:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176915.960654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176916.536622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176916.536628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176916.536632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176916.536633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176916.536635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176916.536636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176916.536693:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176916.536697:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176916.536700:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176916.984657:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176916.984680:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176916.984681:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176916.984685:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176916.984688:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176916.984691:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176916.984693:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176916.984695:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176916.984697:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176917.560581:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176917.560585:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176917.560589:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176917.560590:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176917.560593:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176917.560593:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176917.560624:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176917.560627:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176917.560630:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176918.008601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176918.008609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176918.008611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176918.008616:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176918.008619:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176918.008622:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176918.008625:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176918.008627:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176918.008630:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176918.584622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176918.584627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176918.584632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176918.584633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176918.584635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176918.584636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176918.584699:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176918.584704:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176918.584708:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176918.776623:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301814477, at 4301814528
00000100:00000040:1.0:1786176918.776623:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301814477, at 4301814528
00000100:00000010:0.0:1786176918.776637:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1360.
00000100:00000010:1.0:1786176918.776641:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b19c0.
00000100:00000001:1.0:1786176919.032620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176919.032626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176919.032627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176919.032631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176919.032634:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176919.032638:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176919.032640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176919.032641:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176919.032644:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176919.608634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176919.608642:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176919.608644:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176919.608647:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176919.608651:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176919.608653:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176919.608655:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176919.608656:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176919.608659:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176919.608660:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176919.608661:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176919.608663:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176919.608664:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176919.608665:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176919.608666:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176919.608667:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176919.608669:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176919.608671:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176919.608733:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176919.608737:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176919.608740:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176920.056622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176920.056629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:3.0:1786176920.056629:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176920.056633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176920.056638:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786176920.056641:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176920.056643:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:1.0:1786176920.056645:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176920.056648:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176920.056650:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176920.056653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176920.056656:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7143
00000100:00000001:1.0:1786176920.056658:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176920.056661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.056664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176920.056667:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176920.056673:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176920.056676:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176920.056685:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.056688:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176920.056690:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.056691:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176920.056694:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176920.056696:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176920.056698:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176920.056700:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176920.056702:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176920.056705:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176920.056707:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.056708:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176920.056709:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.056711:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176920.056713:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176920.056715:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176920.056716:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176920.056719:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176920.056721:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176920.056723:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.056725:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176920.056726:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176920.056734:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d180.
00000020:00000040:3.0:1786176920.056736:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176920.056740:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176920.056741:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.056743:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176920.056745:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176920.056749:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176920.056755:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786176920.056761:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176920.056767:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176920.056776:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176920.056779:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176920.056783:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085df314e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176920.056798:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176920.056800:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176920.056803:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176920.056804:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176920.056806:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176920.056808:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176920.056810:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7143
00000400:00000200:3.0:1786176920.056814:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176920.056816:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176920.056818:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176920.056820:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1786176920.056821:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176920.056825:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176920.056825:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176920.056830:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176920.056830:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.056835:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176920.056837:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176920.056838:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786176920.056840:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000040:1.0:1786176920.056842:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786176920.056842:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786176920.056846:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176920.056850:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786176920.056857:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.056860:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176920.056864:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176920.056865:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176920.056866:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176920.056869:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786176920.056870:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786176920.056876:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:1.0:1786176920.056877:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1786176920.056878:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176920.056881:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176920.056881:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176920.056886:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176920.056889:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.056890:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176920.056893:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786176920.056895:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1786176920.056896:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.056902:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176920.056906:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856314112:192.168.202.110@tcp:250:kworker.0
00000100:00000010:3.0:1786176920.056907:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e300.
00000020:00000040:3.0:1786176920.056911:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786176920.056916:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176920.056916:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176920.056917:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.056920:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176920.056921:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786176920.056923:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786176920.056923:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176920.056929:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176920.056929:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786176920.056935:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176920.056936:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:3.0:1786176920.056941:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:1.0:1786176920.056946:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
00000100:00000001:3.0:1786176920.056948:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1786176920.056950:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.056951:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176920.056957:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e3752a60] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176920.056962:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176920.056968:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176920.056970:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00080000:3.0:1786176920.056971:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176920.056976:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786176920.056977:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176920.056978:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1786176920.056980:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000001:3.0:1786176920.056981:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176920.056985:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7143
00000400:00000200:3.0:1786176920.056988:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:1.0:1786176920.056990:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856314112, portal 25
00000400:00000200:3.0:1786176920.056991:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176920.056994:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176920.056998:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000040:3.0:1786176920.056998:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176920.057003:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297088 : -114547945254528 : ffff97d1ba69d180)
00000100:00000001:3.0:1786176920.057003:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176920.057006:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.057010:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176920.057013:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176920.057014:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000100:00000040:1.0:1786176920.057016:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786176920.057016:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176920.057019:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786176920.057022:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786176920.057026:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786176920.057033:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.057035:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176920.057038:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.057042:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176920.057047:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786176920.057053:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176920.057055:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786176920.057056:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176920.057059:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1786176920.057061:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786176920.057063:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:1.0:1786176920.057065:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
02000000:00000001:3.0:1786176920.057066:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.057070:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000200:1.0:1786176920.057071:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856314112, offset 0
00000008:00100000:3.0:1786176920.057072:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176920.057076:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.057078:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176920.057081:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000010:3.0:1786176920.057083:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c700.
00000020:00000040:3.0:1786176920.057086:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176920.057088:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176920.057090:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176920.057091:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176920.057093:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176920.057096:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176920.057097:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176920.057099:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000010:3.0:1786176920.057102:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000400:00000200:1.0:1786176920.057105:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176920.057110:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176920.057114:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1786176920.057114:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176920.057118:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176920.057122:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254625:254625:256:4294967295] 192.168.202.110@tcp LPNI seq info [254614:254614:8:4294967295]
00000100:00100000:3.0:1786176920.057125:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176920.057129:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176920.057145:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786176920.057151:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176920.057155:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176920.057156:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786176920.057160:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e56458ae] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:1.0:1786176920.057162:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000100:00000001:3.0:1786176920.057170:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176920.057174:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786176920.057175:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.0:1786176920.057180:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00080000:3.0:1786176920.057181:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786176920.057184:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176920.057187:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176920.057189:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176920.057192:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7143
00000400:00000200:3.0:1786176920.057197:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:1.1:1786176920.057198:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:3.0:1786176920.057202:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:1.1:1786176920.057203:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00080000:3.0:1786176920.057205:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000100:1.0:1786176920.057211:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786176920.057212:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000010:1.0:1786176920.057215:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000100:00000001:3.0:1786176920.057216:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176920.057221:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.057223:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176920.057224:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:3.0:1786176920.057227:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176920.057230:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000400:00000200:1.0:1786176920.057232:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:3.0:1786176920.057233:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786176920.057237:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000020:00000040:3.0:1786176920.057237:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000400:00000010:1.0:1786176920.057242:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000020:00000001:3.0:1786176920.057242:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786176920.057247:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176920.057249:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:3.0:1786176920.057252:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.057254:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176920.057257:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.057259:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:1.0:1786176920.057260:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786176920.057262:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786176920.057267:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176920.057272:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176920.057276:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176920.057280:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1786176920.057280:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1786176920.057286:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176920.057287:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.057292:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176920.057296:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176920.057303:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.057308:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.057309:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.057312:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000010:3.0:1786176920.057314:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c000.
00000100:00000001:1.0:1786176920.057317:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786176920.057319:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000400:00000200:1.0:1786176920.057323:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176920.057326:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301819597
00000100:00000010:1.0:1786176920.057330:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b19c0.
02000000:00000001:3.0:1786176920.057333:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176920.057335:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:3.0:1786176920.057336:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.057341:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176920.057342:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176920.057344:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.057348:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301819597
02000000:00000010:3.0:1786176920.057349:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000010:1.0:1786176920.057355:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b17e0.
00000100:00000001:3.0:1786176920.057355:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176920.057361:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176920.057364:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1786176920.057369:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176920.057372:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176920.057373:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.057376:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.057378:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:3.0:1786176920.057381:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006fe8fdb7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176920.057385:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176920/real 1786176920]  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176920.057403:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176920.057406:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00080000:3.0:1786176920.057406:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:1.0:1786176920.057408:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176920.057412:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000100:00080000:3.0:1786176920.057412:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1786176920.057417:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000100:00000001:3.0:1786176920.057417:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176920.057420:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1786176920.057421:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:3.0:1786176920.057426:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176920.057427:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176920.057432:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176920.057433:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7143
00000400:00000200:3.0:1786176920.057451:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176920.057457:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176920.057459:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:1.0:1786176920.057462:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176920.057464:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.057470:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176920.057474:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176920.057477:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176920.057478:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176920.057481:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:3.0:1786176920.057482:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176920.057485:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:3.0:1786176920.057486:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:1.0:1786176920.057489:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:3.0:1786176920.057490:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176920.057493:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00100000:1.0:1786176920.057498:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786176920.057498:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786176920.057502:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176920.057529:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176920.057535:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176920.057537:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176920.057542:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176920.057547:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176920.057548:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176920.057551:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176920.057552:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:3.0:1786176920.057554:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:1.0:1786176920.057555:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176920.057559:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176920.057561:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176920.057562:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176920.057565:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786176920.057566:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786176920.057569:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:3.0:1786176920.057570:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176920.057574:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1786176920.057575:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
02000000:00000001:3.0:1786176920.057579:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.057580:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1786176920.057583:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:1.0:1786176920.057587:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1786176920.057587:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176920.057593:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176920.057597:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176920.057603:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ea00.
00000100:00100000:1.0:1786176920.057604:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856314112:192.168.202.110@tcp:250:kworker.0
00000020:00000040:3.0:1786176920.057607:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176920.057610:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786176920.057611:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176920.057613:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1786176920.057614:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176920.057618:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d180 x1872934856314112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176920.057619:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176920.057622:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176920.057624:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176920.057629:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:1.0:1786176920.057630:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176920.057634:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:3.0:1786176920.057634:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176920.057638:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
00000100:00100000:3.0:1786176920.057642:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786176920.057643:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176920.057646:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176920.057649:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176920.057653:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00000001:3.0:1786176920.057655:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000010:1.0:1786176920.057656:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786176920.057659:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176920.057661:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176920.057665:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000040:3.0:1786176920.057665:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053162887] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1786176920.057669:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d180.
00000100:00000001:1.0:1786176920.057674:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176920.057676:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.057678:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176920.057678:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176920.057684:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7153)
00000100:00000001:1.0:1786176920.057686:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176920.057689:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.057691:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176920.057692:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176920.057693:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176920.057697:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.057702:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176920.057704:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176920.057705:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176920.057707:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176920.057708:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176920.057710:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856314240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176920.057719:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176920.057721:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176920.057722:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176920.057723:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176920.057726:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176920.057727:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176920.057730:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176920.057732:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176920.057735:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176920.057737:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856314240, portal 10
00000100:00000001:1.0:1786176920.057738:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176920.057739:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:1.0:1786176920.057744:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.057749:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176920.057750:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176920.057752:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176920.057754:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856314240, offset 0
00000400:00000200:1.0:1786176920.057757:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176920.057760:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176920.057762:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176920.057765:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176920.057767:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176920.057769:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176920.057770:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176920.057773:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254626:254626:256:4294967295] 192.168.202.110@tcp LPNI seq info [254615:254615:8:4294967295]
00000400:00000200:1.0:1786176920.057775:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176920.057780:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176920.057783:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176920.057785:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176920.057788:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176920.057790:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786176920.057793:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176920.057795:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786176920.057798:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176920.057799:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176920.057801:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176920.057803:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176920.057805:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176920.057807:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176920.057810:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176920.057814:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.057822:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176920.057825:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.057832:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.057834:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176920.057837:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176920.057842:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176920.057845:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176920.057847:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301815818
00000100:00000001:1.0:1786176920.057849:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301819597
00000100:00000001:1.0:1786176920.057851:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301815818
00000100:00000001:1.0:1786176920.057853:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301819597
00000100:00000001:1.0:1786176920.057855:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176920.057857:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.057859:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176920.057862:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176920/real 1786176920]  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176920.057869:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176920.057871:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176920.057872:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176920.057874:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176920.057876:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176920.057878:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176920.057881:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176920.057889:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.057895:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176920.057896:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.057898:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176920.057899:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176920.057903:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.057909:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176920.057912:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.057919:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176920.057921:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.057923:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176920.057924:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.057926:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176920.057928:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176920.057931:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176920.057933:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176920.057935:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176920.057939:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176920.057946:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856314240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176920.057950:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176920.057953:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.057958:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176920.057959:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176920.057961:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176920.057963:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176920.057965:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176920.057967:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856314368:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176920.057970:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176920.057972:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176920.057973:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176920.057975:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176920.057978:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176920.057980:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176920.057983:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176920.057985:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176920.057988:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176920.057992:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856314368, portal 10
00000100:00000001:1.0:1786176920.057994:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176920.057995:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:1.0:1786176920.058001:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058007:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176920.058009:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176920.058012:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176920.058014:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856314368, offset 0
00000400:00000200:1.0:1786176920.058018:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176920.058022:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176920.058025:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176920.058028:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176920.058031:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176920.058033:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176920.058035:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176920.058038:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254627:254627:256:4294967295] 192.168.202.110@tcp LPNI seq info [254616:254616:8:4294967295]
00000400:00000200:1.0:1786176920.058042:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176920.058048:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176920.058053:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176920.058056:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176920.058060:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176920.058063:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786176920.058068:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176920.058071:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786176920.058073:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176920.058075:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176920.058077:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176920.058079:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176920.058082:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176920.058084:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176920.058085:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176920.058088:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058094:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176920.058096:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058100:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.058101:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176920.058103:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176920.058105:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176920.058108:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176920.058109:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301815943
00000100:00000001:1.0:1786176920.058111:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301819597
00000100:00000001:1.0:1786176920.058112:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301815943
00000100:00000001:1.0:1786176920.058114:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301819597
00000100:00000001:1.0:1786176920.058115:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176920.058116:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.058117:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176920.058119:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176920/real 1786176920]  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176920.058124:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176920.058125:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176920.058127:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176920.058128:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176920.058129:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176920.058131:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176920.058133:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176920.058138:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058143:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176920.058144:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058145:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176920.058146:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176920.058148:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058152:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176920.058154:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058158:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176920.058160:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058161:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176920.058162:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058163:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176920.058164:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176920.058165:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176920.058167:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176920.058168:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176920.058171:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176920.058175:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856314368:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176920.058178:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176920.058180:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058184:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176920.058185:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176920.058187:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176920.058188:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176920.058189:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176920.058191:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856314496:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176920.058193:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176920.058195:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176920.058195:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176920.058196:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176920.058199:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176920.058200:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176920.058202:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176920.058204:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176920.058206:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176920.058208:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856314496, portal 4
00000100:00000001:1.0:1786176920.058209:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176920.058210:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:1.0:1786176920.058214:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058218:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176920.058220:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176920.058221:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176920.058223:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856314496, offset 0
00000400:00000200:1.0:1786176920.058225:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176920.058227:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176920.058229:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176920.058231:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176920.058233:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176920.058235:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176920.058236:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176920.058239:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254628:254628:256:4294967295] 192.168.202.110@tcp LPNI seq info [254617:254617:8:4294967295]
00000400:00000200:1.0:1786176920.058241:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176920.058245:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176920.058248:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176920.058250:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176920.058252:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176920.058254:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786176920.058256:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176920.058258:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786176920.058261:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176920.058262:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176920.058264:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176920.058266:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176920.058268:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176920.058269:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176920.058270:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176920.058272:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058277:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176920.058279:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058283:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.058284:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176920.058285:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176920.058288:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176920.058290:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176920.058291:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301815818
00000100:00000001:1.0:1786176920.058292:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301819597
00000100:00000001:1.0:1786176920.058294:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301815818
00000100:00000001:1.0:1786176920.058295:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301819597
00000100:00000001:1.0:1786176920.058296:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176920.058297:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.058299:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176920.058301:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176920/real 1786176920]  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176920.058306:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176920.058307:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176920.058308:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176920.058309:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176920.058310:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176920.058311:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176920.058313:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176920.058318:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058322:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176920.058323:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058324:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176920.058325:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176920.058327:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058331:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176920.058333:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058337:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176920.058339:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058340:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176920.058341:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058342:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176920.058342:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176920.058344:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176920.058346:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176920.058347:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176920.058350:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176920.058355:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856314496:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176920.058357:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176920.058359:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058363:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176920.058364:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176920.058365:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176920.058366:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176920.058367:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176920.058369:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856314624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176920.058370:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176920.058371:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176920.058372:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176920.058373:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176920.058375:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176920.058376:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176920.058378:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176920.058380:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176920.058381:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176920.058383:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856314624, portal 4
00000100:00000001:1.0:1786176920.058385:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176920.058385:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:1.0:1786176920.058389:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058392:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176920.058394:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176920.058395:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176920.058397:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856314624, offset 0
00000400:00000200:1.0:1786176920.058400:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176920.058401:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176920.058403:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176920.058405:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176920.058407:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176920.058408:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176920.058410:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176920.058412:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254629:254629:256:4294967295] 192.168.202.110@tcp LPNI seq info [254618:254618:8:4294967295]
00000400:00000200:1.0:1786176920.058415:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176920.058418:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176920.058421:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176920.058422:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176920.058425:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176920.058426:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786176920.058429:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176920.058431:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786176920.058433:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176920.058435:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176920.058437:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176920.058453:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176920.058456:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176920.058457:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176920.058458:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176920.058461:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058466:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176920.058468:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176945 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058472:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.058473:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176920.058475:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176920.058477:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176920.058479:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176920.058481:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301815826
00000100:00000001:1.0:1786176920.058483:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301819597
00000100:00000001:1.0:1786176920.058485:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301815826
00000100:00000001:1.0:1786176920.058486:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301819597
00000100:00000001:1.0:1786176920.058487:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176920.058488:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.058489:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176920.058492:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176920/real 1786176920]  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176920.058496:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176920.058497:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176920.058498:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176920.058499:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176920.058501:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176920.058502:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176920.058504:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176920.058523:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058528:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176920.058529:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058531:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176920.058532:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176920.058534:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058539:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176920.058541:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058545:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176920.058547:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058548:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176920.058549:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058551:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176920.058552:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176920.058554:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176920.058555:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176920.058557:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176920.058559:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176920.058563:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856314624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176920.058566:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058567:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176920.058569:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856314240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058573:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176920.058574:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176920.058575:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176920.058577:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176920.058578:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176920.058579:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176920.058580:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176920.058582:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:1.0:1786176920.058584:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176920.058584:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176920.058586:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:1.0:1786176920.058587:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176920.058588:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058590:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176920.058592:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856314368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058600:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176920.058601:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176920.058602:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176920.058604:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176920.058604:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176920.058605:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176920.058606:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176920.058607:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176920.058608:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176920.058609:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176920.058610:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:1.0:1786176920.058611:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176920.058612:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058612:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176920.058614:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856314496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058618:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176920.058619:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176920.058620:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176920.058621:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176920.058621:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176920.058622:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176920.058623:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176920.058624:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786176920.058626:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176920.058626:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176920.058627:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:1.0:1786176920.058629:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176920.058630:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058631:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176920.058634:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856314624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176945 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176920.058638:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176920.058639:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176920.058640:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176920.058643:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176920.058645:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176920.058646:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176920.058647:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176920.058648:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176920.058650:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176920.058651:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176920.058652:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:1.0:1786176920.058653:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176920.058654:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058655:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176920.058658:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176920.058659:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176920.058660:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176920.058661:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176920.632622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176920.632628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176920.632632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176920.632633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176920.632635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176920.632637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176920.632689:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176920.632692:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176920.632694:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176921.080612:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176921.080622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176921.080624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176921.080631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176921.080635:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176921.080640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176921.080643:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176921.080647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176921.080650:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176921.656599:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176921.656604:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176921.656608:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176921.656609:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176921.656612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176921.656613:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176921.656672:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176921.656678:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176921.656682:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176922.104585:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176922.104591:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176922.104592:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176922.104597:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176922.104599:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176922.104603:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176922.104605:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176922.104607:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176922.104609:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176922.680619:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176922.680625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176922.680629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176922.680630:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176922.680632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176922.680635:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176922.680637:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176922.680639:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176922.680642:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176922.680643:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176922.680645:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176922.680647:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176922.680700:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176922.680704:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176922.680707:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176923.128648:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176923.128657:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176923.128659:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176923.128664:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176923.128668:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176923.128673:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176923.128676:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176923.128679:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176923.128682:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176923.704648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176923.704656:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176923.704663:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176923.704664:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176923.704668:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176923.704669:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176923.704745:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176923.704749:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176923.704752:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176923.896583:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301819597, at 4301819648
00000100:00000040:1.0:1786176923.896590:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301819597, at 4301819648
00000100:00000010:0.0:1786176923.896596:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b19c0.
00000100:00000010:1.0:1786176923.896598:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b17e0.
00000100:00000001:1.0:1786176924.152628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176924.152634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176924.152636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176924.152641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176924.152644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176924.152647:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176924.152649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176924.152651:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176924.152653:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176924.728694:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176924.728704:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176924.728712:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176924.728714:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176924.728717:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176924.728719:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176924.728821:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176924.728829:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176924.728834:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176925.176608:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1786176925.176608:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176925.176614:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:3.0:1786176925.176617:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1786176925.176618:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.176622:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:1.0:1786176925.176624:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:3.0:1786176925.176626:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176925.176629:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176925.176630:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176925.176634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176925.176638:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7148
00000100:00000001:1.0:1786176925.176639:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176925.176642:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.176646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176925.176649:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176925.176655:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176925.176658:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176925.176661:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.176664:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176925.176667:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.176669:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176925.176672:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176925.176674:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176925.176676:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176925.176679:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176925.176680:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176925.176682:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176925.176684:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.176685:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176925.176687:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.176688:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176925.176690:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176925.176692:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176925.176694:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176925.176696:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176925.176699:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176925.176701:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.176702:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176925.176703:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176925.176710:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d500.
00000020:00000040:3.0:1786176925.176713:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176925.176716:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176925.176717:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.176719:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176925.176722:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176925.176725:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176925.176731:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:3.0:1786176925.176736:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176925.176743:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176925.176751:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176925.176754:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176925.176758:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7c36393] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176925.176780:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176925.176783:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176925.176785:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176925.176786:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176925.176787:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176925.176789:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176925.176791:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7148
00000400:00000200:3.0:1786176925.176794:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176925.176796:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176925.176797:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176925.176799:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.176800:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176925.176802:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.176804:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176925.176806:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176925.176808:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786176925.176810:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176925.176812:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786176925.176815:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176925.176818:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786176925.176822:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.176824:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176925.176827:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1786176925.176828:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.176833:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176925.176834:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176925.176838:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1786176925.176839:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786176925.176845:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:1.0:1786176925.176850:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176925.176850:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176925.176856:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176925.176863:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176925.176867:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1786176925.176867:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.176868:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176925.176872:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1786176925.176873:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176925.176877:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176925.176878:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1786176925.176879:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176925.176884:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:3.0:1786176925.176885:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f100.
00000020:00000040:3.0:1786176925.176889:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786176925.176891:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176925.176893:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176925.176895:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.176897:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176925.176900:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:1.0:1786176925.176901:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856314752:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1786176925.176902:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.176909:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176925.176914:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000010:3.0:1786176925.176915:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:1.0:1786176925.176918:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.176920:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176925.176923:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:3.0:1786176925.176928:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:1.0:1786176925.176931:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176925.176936:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.176938:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176925.176942:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176925.176945:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1ef9f45] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176925.176949:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176925.176957:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00000001:3.0:1786176925.176958:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176925.176964:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176925.176967:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:1.0:1786176925.176968:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000001:3.0:1786176925.176969:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176925.176972:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176925.176975:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176925.176977:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7148
00000100:00000200:1.0:1786176925.176979:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856314752, portal 25
00000400:00000200:3.0:1786176925.176982:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176925.176986:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000400:00000200:3.0:1786176925.176987:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176925.176990:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000100:00080000:3.0:1786176925.176991:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176925.176996:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.176999:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176925.177003:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176925.177004:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176925.177005:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176925.177009:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176925.177011:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786176925.177013:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176925.177015:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786176925.177019:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786176925.177023:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:1.0:1786176925.177028:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176925.177034:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786176925.177036:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.177039:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:1.0:1786176925.177040:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000040:3.0:1786176925.177041:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.177046:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:1.0:1786176925.177049:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856314752, offset 0
00000020:00000040:3.0:1786176925.177049:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786176925.177054:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176925.177057:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176925.177058:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176925.177061:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786176925.177066:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786176925.177066:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.177070:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176925.177072:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176925.177074:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.177075:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176925.177078:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f480.
00000400:00000200:1.0:1786176925.177079:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:3.0:1786176925.177081:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176925.177085:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176925.177086:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.177088:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176925.177089:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176925.177091:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176925.177092:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000010:3.0:1786176925.177096:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:3.0:1786176925.177100:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176925.177101:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:3.0:1786176925.177106:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176925.177109:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176925.177117:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1786176925.177118:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176925.177122:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176925.177124:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:3.0:1786176925.177128:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092aa1aa7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176925.177135:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254630:254630:256:4294967295] 192.168.202.110@tcp LPNI seq info [254619:254619:8:4294967295]
00000100:00000001:3.0:1786176925.177142:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176925.177145:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786176925.177148:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1786176925.177149:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786176925.177153:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176925.177154:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176925.177157:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176925.177162:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7148
00000400:00000200:3.0:1786176925.177169:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176925.177174:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1786176925.177174:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176925.177179:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176925.177184:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.177186:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176925.177189:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:1.0:1786176925.177191:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176925.177191:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176925.177197:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176925.177198:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786176925.177199:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176925.177201:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000800:00000010:1.0:1786176925.177202:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000020:00000001:3.0:1786176925.177203:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786176925.177209:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786176925.177211:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.177213:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176925.177217:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:1.0:1786176925.177218:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786176925.177219:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176925.177222:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786176925.177226:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:1.0:1786176925.177227:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:3.0:1786176925.177228:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176925.177230:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176925.177232:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176925.177235:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.177237:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176925.177239:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000800:00000200:1.1:1786176925.177242:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000008:00000001:3.0:1786176925.177243:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.177245:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176925.177251:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ca80.
00000800:00000200:1.1:1786176925.177251:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000040:3.0:1786176925.177255:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176925.177258:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176925.177260:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.177263:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000800:00000100:1.0:1786176925.177264:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786176925.177265:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176925.177268:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786176925.177271:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
02000000:00000010:3.0:1786176925.177278:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000400:00000200:1.0:1786176925.177279:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1786176925.177281:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176925.177288:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176925.177290:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1786176925.177299:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176925.177300:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000100:00000001:3.0:1786176925.177301:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176925.177306:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ffb0865] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176925.177311:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176925.177313:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176925.177316:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000010:1.0:1786176925.177317:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:3.0:1786176925.177317:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176925.177320:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176925.177323:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176925.177325:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:3.0:1786176925.177326:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7148
00000400:00000200:3.0:1786176925.177333:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:1.0:1786176925.177335:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786176925.177338:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176925.177341:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176925.177347:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.177351:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176925.177354:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.177358:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176925.177361:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176925.177362:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:1.0:1786176925.177363:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000001:3.0:1786176925.177363:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176925.177368:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786176925.177370:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:1.0:1786176925.177372:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786176925.177373:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786176925.177380:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176925.177384:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176925.177388:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176925.177410:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.177413:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176925.177417:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176925.177419:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:1.0:1786176925.177422:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176925.177422:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176925.177427:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176925.177429:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786176925.177432:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786176925.177432:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176925.177437:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.177439:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176925.177441:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301824717
00000008:00100000:3.0:1786176925.177442:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176925.177447:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176925.177449:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786176925.177452:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b17e0.
00000100:00000010:3.0:1786176925.177453:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69fb80.
00000020:00000040:3.0:1786176925.177457:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176925.177459:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176925.177460:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.177462:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786176925.177462:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176925.177465:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176925.177467:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176925.177470:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:1.0:1786176925.177471:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301824717
00000100:00000001:3.0:1786176925.177473:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176925.177483:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:1.0:1786176925.177487:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1960.
00000100:00000001:3.0:1786176925.177490:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176925.177493:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.177494:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:3.0:1786176925.177496:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014f55bbb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.177502:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176925.177506:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.177507:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176925.177528:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7158)
00000100:00000001:1.0:1786176925.177551:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176925.177557:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176925/real 1786176925]  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176925.177568:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176925.177570:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176925.177573:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176925.177576:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176925.177579:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176925.177581:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176925.177586:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176925.177596:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.177602:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176925.177604:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.177607:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176925.177609:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176925.177614:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.177621:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176925.177626:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.177635:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176925.177638:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.177640:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176925.177641:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.177644:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176925.177646:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176925.177649:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176925.177653:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176925.177656:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176925.177660:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176925.177678:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856314752:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176925.177683:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.177686:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176925.177689:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856314752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.177703:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176925.177706:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176925.177708:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176925.177713:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176925.177715:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176925.177717:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176925.177719:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176925.177721:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:1.0:1786176925.177726:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176925.177727:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176925.177729:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:1.0:1786176925.177733:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176925.177734:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.177736:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.177743:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176925.177744:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.177747:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176925.177748:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176925.177750:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176925.177755:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.177763:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176925.177765:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176925.177767:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176925.177769:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176925.177771:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176925.177774:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856314880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176925.177778:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176925.177780:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176925.177781:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176925.177783:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176925.177786:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176925.177789:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176925.177793:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176925.177796:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176925.177800:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176925.177803:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856314880, portal 10
00000100:00000001:1.0:1786176925.177805:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176925.177807:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:1.0:1786176925.177814:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.177821:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176925.177823:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176925.177826:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176925.177828:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856314880, offset 0
00000400:00000200:1.0:1786176925.177833:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176925.177837:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176925.177841:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176925.177845:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176925.177848:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176925.177850:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176925.177853:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176925.177856:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254631:254631:256:4294967295] 192.168.202.110@tcp LPNI seq info [254620:254620:8:4294967295]
00000400:00000200:1.0:1786176925.177860:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176925.177867:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176925.177872:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176925.177876:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176925.177880:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176925.177883:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786176925.177888:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176925.177891:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1786176925.177895:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176925.177897:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176925.177901:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176925.177904:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176925.177908:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176925.177910:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176925.177913:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176925.177917:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.177925:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176925.177928:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.177935:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.177937:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176925.177940:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176925.177944:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176925.177947:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176925.177949:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301820938
00000100:00000001:1.0:1786176925.177952:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301824717
00000100:00000001:1.0:1786176925.177955:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301820938
00000100:00000001:1.0:1786176925.177957:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301824717
00000100:00000001:1.0:1786176925.177959:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176925.177961:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.177963:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176925.177967:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176925/real 1786176925]  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176925.177975:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176925.177976:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176925.177978:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176925.177980:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176925.177982:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176925.177984:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176925.177987:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176925.177996:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178008:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176925.178009:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.178011:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176925.178013:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176925.178017:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178024:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176925.178029:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178036:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176925.178039:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.178041:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176925.178042:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.178045:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176925.178047:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176925.178049:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176925.178053:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176925.178055:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176925.178059:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176925.178067:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856314880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176925.178071:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176925.178075:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178082:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176925.178083:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176925.178086:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176925.178088:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176925.178090:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176925.178093:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176925.178097:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176925.178099:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176925.178100:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176925.178102:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176925.178105:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176925.178108:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176925.178111:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176925.178114:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176925.178117:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176925.178120:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856315008, portal 10
00000100:00000001:1.0:1786176925.178122:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176925.178123:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:1.0:1786176925.178130:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178137:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176925.178139:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176925.178141:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176925.178144:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856315008, offset 0
00000400:00000200:1.0:1786176925.178148:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176925.178152:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176925.178157:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176925.178160:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176925.178162:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176925.178165:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176925.178167:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176925.178170:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254632:254632:256:4294967295] 192.168.202.110@tcp LPNI seq info [254621:254621:8:4294967295]
00000400:00000200:1.0:1786176925.178174:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176925.178180:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176925.178185:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176925.178189:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176925.178193:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176925.178195:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786176925.178200:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176925.178202:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1786176925.178207:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176925.178208:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176925.178212:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176925.178215:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176925.178219:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176925.178221:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176925.178223:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176925.178227:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178235:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176925.178238:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178245:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.178247:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176925.178250:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176925.178253:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176925.178256:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176925.178258:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301821063
00000100:00000001:1.0:1786176925.178261:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301824717
00000100:00000001:1.0:1786176925.178263:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301821063
00000100:00000001:1.0:1786176925.178265:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301824717
00000100:00000001:1.0:1786176925.178267:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176925.178269:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.178271:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176925.178275:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176925/real 1786176925]  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176925.178283:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176925.178284:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176925.178286:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176925.178289:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176925.178290:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176925.178292:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176925.178296:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176925.178305:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178311:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176925.178313:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.178315:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176925.178316:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176925.178320:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178327:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176925.178331:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178339:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176925.178341:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.178343:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176925.178344:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.178346:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176925.178348:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176925.178350:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176925.178353:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176925.178355:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176925.178360:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176925.178368:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176925.178372:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176925.178375:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178382:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176925.178384:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176925.178386:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176925.178388:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176925.178405:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176925.178409:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176925.178412:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176925.178414:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176925.178416:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176925.178418:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176925.178422:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176925.178423:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176925.178427:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176925.178430:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176925.178434:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176925.178437:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856315136, portal 4
00000100:00000001:1.0:1786176925.178439:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176925.178440:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:1.0:1786176925.178446:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178453:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176925.178455:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176925.178457:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176925.178460:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856315136, offset 0
00000400:00000200:1.0:1786176925.178464:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176925.178468:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176925.178472:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176925.178476:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176925.178478:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176925.178481:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176925.178483:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176925.178486:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254633:254633:256:4294967295] 192.168.202.110@tcp LPNI seq info [254622:254622:8:4294967295]
00000400:00000200:1.0:1786176925.178490:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176925.178497:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176925.178502:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176925.178506:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176925.178528:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176925.178532:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786176925.178536:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176925.178539:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1786176925.178543:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176925.178545:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176925.178548:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176925.178552:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176925.178556:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176925.178557:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176925.178560:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176925.178564:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178572:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176925.178576:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178582:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.178584:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176925.178587:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176925.178590:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176925.178593:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176925.178596:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301820939
00000100:00000001:1.0:1786176925.178598:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301824717
00000100:00000001:1.0:1786176925.178601:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301820939
00000100:00000001:1.0:1786176925.178603:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301824717
00000100:00000001:1.0:1786176925.178605:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176925.178607:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.178609:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176925.178613:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176925/real 1786176925]  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176925.178621:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176925.178622:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176925.178624:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176925.178627:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176925.178629:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176925.178631:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176925.178634:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176925.178643:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178650:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176925.178651:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.178653:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176925.178654:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176925.178658:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178665:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176925.178669:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178676:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176925.178679:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.178680:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176925.178682:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.178684:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176925.178686:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176925.178688:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176925.178692:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176925.178694:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176925.178698:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176925.178706:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176925.178710:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176925.178714:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178721:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176925.178722:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176925.178724:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176925.178726:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176925.178728:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176925.178731:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176925.178734:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176925.178736:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176925.178738:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176925.178740:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176925.178744:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176925.178746:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176925.178749:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176925.178752:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176925.178755:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176925.178759:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856315264, portal 4
00000100:00000001:1.0:1786176925.178761:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176925.178763:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000040:1.0:1786176925.178768:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178775:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176925.178777:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176925.178780:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176925.178782:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856315264, offset 0
00000400:00000200:1.0:1786176925.178787:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176925.178791:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176925.178794:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176925.178797:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176925.178800:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176925.178802:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176925.178805:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176925.178808:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254634:254634:256:4294967295] 192.168.202.110@tcp LPNI seq info [254623:254623:8:4294967295]
00000400:00000200:1.0:1786176925.178812:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176925.178818:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176925.178823:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176925.178826:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176925.178830:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176925.178833:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786176925.178837:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176925.178840:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1786176925.178844:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176925.178846:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176925.178849:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176925.178853:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176925.178856:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176925.178857:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176925.178860:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176925.178864:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178877:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176925.178880:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178887:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.178888:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176925.178891:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176925.178894:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176925.178897:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176925.178900:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301820947
00000100:00000001:1.0:1786176925.178902:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301824717
00000100:00000001:1.0:1786176925.178904:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301820947
00000100:00000001:1.0:1786176925.178907:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301824717
00000100:00000001:1.0:1786176925.178909:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176925.178911:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.178913:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176925.178917:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176925/real 1786176925]  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176925.178925:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176925.178927:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176925.178928:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176925.178931:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176925.178933:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176925.178935:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176925.178938:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176925.178947:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178955:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176925.178957:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.178959:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176925.178960:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176925.178964:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178971:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176925.178975:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.178983:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176925.178985:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.178986:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176925.178988:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.178990:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176925.178992:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176925.178994:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176925.178998:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176925.179000:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176925.179004:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176925.179012:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176925.179016:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.179017:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176925.179020:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856314880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.179027:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176925.179029:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176925.179031:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176925.179034:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176925.179035:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176925.179037:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176925.179039:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176925.179041:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:1.0:1786176925.179044:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176925.179046:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176925.179047:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:1.0:1786176925.179051:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176925.179052:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.179054:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176925.179058:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856315008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.179064:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176925.179066:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176925.179067:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176925.179070:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176925.179071:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176925.179072:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176925.179074:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176925.179076:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176925.179079:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176925.179080:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176925.179081:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:1.0:1786176925.179083:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176925.179085:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.179087:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176925.179090:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856315136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.179097:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176925.179099:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176925.179100:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176925.179102:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176925.179103:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176925.179105:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176925.179107:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176925.179108:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786176925.179110:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176925.179112:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176925.179113:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:1.0:1786176925.179116:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176925.179117:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.179119:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176925.179122:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856315264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176925.179129:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176925.179130:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176925.179132:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176925.179134:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176925.179136:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176925.179137:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176925.179139:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176925.179141:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176925.179143:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176925.179144:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176925.179146:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:1.0:1786176925.179148:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176925.179150:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.179151:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176925.179155:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176925.179156:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176925.179158:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176925.179160:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176925.752603:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176925.752608:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176925.752612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176925.752613:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176925.752616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176925.752617:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176925.752679:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176925.752683:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176925.752686:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176926.200663:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176926.200670:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176926.200672:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176926.200676:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176926.200679:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176926.200682:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176926.200684:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176926.200687:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176926.200689:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176926.776607:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176926.776620:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176926.776624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176926.776625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176926.776628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176926.776629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176926.776709:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176926.776714:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176926.776718:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176927.224630:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176927.224636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176927.224638:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176927.224643:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176927.224645:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176927.224648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176927.224650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176927.224653:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176927.224654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176927.800628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176927.800633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176927.800637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176927.800638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176927.800641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176927.800642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176927.800696:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176927.800699:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176927.800701:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176928.248588:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176928.248596:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176928.248598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176928.248604:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176928.248607:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176928.248611:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176928.248614:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176928.248616:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176928.248619:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176928.824600:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176928.824605:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176928.824609:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176928.824611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176928.824613:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176928.824614:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176928.824658:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176928.824662:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176928.824665:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176929.016618:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301824717, at 4301824768
00000100:00000040:1.0:1786176929.016618:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301824717, at 4301824768
00000100:00000010:1.0:1786176929.016633:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1960.
00000100:00000010:0.0:1786176929.016635:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b17e0.
00000100:00000001:1.0:1786176929.272641:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176929.272648:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176929.272650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176929.272655:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176929.272658:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176929.272662:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176929.272664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176929.272666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176929.272668:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176929.848604:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176929.848613:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176929.848615:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176929.848620:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176929.848625:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176929.848628:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176929.848630:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176929.848632:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176929.848637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176929.848638:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176929.848640:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176929.848641:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176929.848643:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176929.848645:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176929.848646:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176929.848649:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176929.848652:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176929.848654:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176929.848707:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176929.848710:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176929.848713:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176930.296603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176930.296609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176930.296611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176930.296616:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176930.296618:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176930.296621:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.296623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176930.296626:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.296628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176930.296701:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176930.296706:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786176930.296707:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176930.296711:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176930.296714:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176930.296720:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7153
00000400:00000200:3.0:1786176930.296728:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176930.296733:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176930.296736:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176930.296738:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.296741:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176930.296743:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.296745:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176930.296748:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176930.296749:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176930.296752:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176930.296754:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176930.296755:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176930.296757:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176930.296758:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.296760:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176930.296761:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.296763:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176930.296773:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176930.296775:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176930.296777:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176930.296779:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176930.296781:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176930.296783:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.296785:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176930.296786:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176930.296794:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f800.
00000020:00000040:3.0:1786176930.296796:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176930.296800:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176930.296801:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.296804:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176930.296806:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176930.296809:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176930.296816:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786176930.296821:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176930.296828:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176930.296837:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176930.296841:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176930.296844:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4d339d9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.296885:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176930.296888:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176930.296890:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176930.296894:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.296900:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176930.296902:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176930.296905:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176930.296908:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176930.296911:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176930.296921:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856315392:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176930.296925:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176930.296929:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176930.296932:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176930.296934:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176930.296940:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176930.296942:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176930.296949:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176930.296954:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176930.296960:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176930.296966:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856315392, portal 25
00000100:00000001:1.0:1786176930.296968:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176930.296970:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
00000100:00000040:1.0:1786176930.296974:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.296980:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176930.296983:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176930.296986:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176930.296989:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856315392, offset 0
00000400:00000200:1.0:1786176930.296997:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176930.297006:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176930.297013:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176930.297019:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176930.297023:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176930.297027:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176930.297030:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176930.297033:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254635:254635:256:4294967295] 192.168.202.110@tcp LPNI seq info [254624:254624:8:4294967295]
00000400:00000200:1.0:1786176930.297037:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176930.297046:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176930.297051:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176930.297055:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176930.297061:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176930.297064:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786176930.297068:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176930.297071:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1786176930.297074:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176930.297076:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176930.297081:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176930.297085:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176930.297089:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176930.297091:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176930.297094:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176930.297097:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176930.297121:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176930.297123:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176930.297126:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176930.297127:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176930.297129:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176930.297131:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176930.297133:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7153
00000400:00000200:3.0:1786176930.297136:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176930.297138:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176930.297140:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176930.297142:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.297143:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176930.297145:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.297146:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176930.297148:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176930.297148:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786176930.297150:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176930.297151:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786176930.297152:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176930.297154:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786176930.297155:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.297157:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176930.297158:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.297159:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176930.297161:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786176930.297163:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176930.297165:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176930.297166:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176930.297171:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176930.297172:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.297174:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176930.297177:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176930.297179:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.297181:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176930.297183:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d880.
00000020:00000040:3.0:1786176930.297185:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176930.297187:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176930.297188:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.297190:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176930.297191:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176930.297192:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176930.297195:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786176930.297198:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176930.297201:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176930.297206:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176930.297208:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176930.297211:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097f9b8db] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297235:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176930.297238:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297244:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.297245:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176930.297250:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176930.297253:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176930.297255:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301829837
00000100:00000010:1.0:1786176930.297259:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1960.
00000100:00000001:1.0:1786176930.297264:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1786176930.297269:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301829837
00000100:00000010:1.0:1786176930.297271:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1cc0.
00000100:00000001:1.0:1786176930.297273:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1786176930.297276:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176930.297277:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.297279:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176930.297282:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176930/real 1786176930]  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176930.297288:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176930.297289:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176930.297291:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176930.297293:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176930.297294:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176930.297296:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176930.297298:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176930.297304:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297308:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176930.297309:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.297310:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176930.297312:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176930.297314:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297318:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176930.297322:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297327:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176930.297329:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.297330:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176930.297332:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.297333:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176930.297334:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176930.297336:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176930.297339:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176930.297354:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176930.297357:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176930.297363:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856315392:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176930.297366:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.297368:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176930.297370:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856315392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297375:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176930.297376:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176930.297378:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176930.297380:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176930.297381:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176930.297383:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176930.297384:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176930.297386:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:1.0:1786176930.297389:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176930.297390:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176930.297391:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000001:1.0:1786176930.297393:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176930.297394:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.297396:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.297402:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176930.297403:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.297404:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176930.297405:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176930.297406:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176930.297409:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297414:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176930.297415:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176930.297416:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176930.297418:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176930.297419:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176930.297421:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315520:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176930.297423:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176930.297424:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176930.297425:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176930.297426:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176930.297428:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176930.297429:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176930.297432:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176930.297433:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176930.297436:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176930.297438:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856315520, portal 10
00000100:00000001:1.0:1786176930.297439:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176930.297441:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
00000100:00000040:1.0:1786176930.297446:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297450:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176930.297452:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176930.297453:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176930.297454:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856315520, offset 0
00000400:00000200:1.0:1786176930.297457:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176930.297460:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176930.297462:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176930.297464:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176930.297466:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176930.297468:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176930.297470:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176930.297474:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254636:254636:256:4294967295] 192.168.202.110@tcp LPNI seq info [254625:254625:8:4294967295]
00000400:00000200:1.0:1786176930.297478:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176930.297485:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176930.297489:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176930.297493:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176930.297497:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176930.297500:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786176930.297505:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176930.297508:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1786176930.297532:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176930.297533:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176930.297536:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176930.297538:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176930.297540:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176930.297542:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176930.297543:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176930.297546:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297552:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176930.297554:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297559:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.297560:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176930.297562:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176930.297566:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176930.297568:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176930.297569:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301826057
00000100:00000001:1.0:1786176930.297571:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301829837
00000100:00000001:1.0:1786176930.297573:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301826057
00000100:00000001:1.0:1786176930.297574:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301829837
00000100:00000001:1.0:1786176930.297576:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176930.297578:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.297579:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176930.297581:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176930/real 1786176930]  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176930.297587:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176930.297588:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176930.297589:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176930.297590:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176930.297591:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176930.297592:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176930.297594:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176930.297600:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297604:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176930.297605:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.297606:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176930.297607:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176930.297609:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297613:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176930.297615:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297620:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176930.297621:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.297622:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176930.297623:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.297624:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176930.297625:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176930.297627:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176930.297629:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176930.297631:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176930.297634:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176930.297638:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315520:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176930.297641:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.297642:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176930.297644:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856315520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.297647:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176930.297648:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176930.297650:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176930.297651:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176930.297652:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176930.297653:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176930.297654:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176930.297655:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:1.0:1786176930.297657:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176930.297657:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176930.297658:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:1.0:1786176930.297660:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176930.297661:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.297662:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.297664:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176930.297665:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.297666:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176930.297667:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.297716:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176930.297719:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176930.297721:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786176930.297723:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176930.297724:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176930.297727:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176930.297730:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7153
00000400:00000200:3.0:1786176930.297734:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176930.297737:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176930.297738:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:3.0:1786176930.297739:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176930.297742:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.297744:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176930.297745:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.297747:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176930.297748:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176930.297750:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176930.297753:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786176930.297754:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176930.297756:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786176930.297758:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786176930.297760:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786176930.297763:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.297765:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176930.297766:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.297767:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176930.297769:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786176930.297771:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176930.297772:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176930.297773:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176930.297775:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176930.297777:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.297778:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176930.297779:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176930.297781:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.297782:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176930.297785:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c380.
00000020:00000040:3.0:1786176930.297787:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176930.297788:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176930.297790:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.297791:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176930.297793:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176930.297794:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176930.297798:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786176930.297800:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176930.297803:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176930.297808:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176930.297810:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176930.297812:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000833bf1c9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176930.297824:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176930.297826:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176930.297828:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786176930.297829:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176930.297830:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176930.297832:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176930.297835:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7153
00000400:00000200:3.0:1786176930.297837:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176930.297840:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176930.297842:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176930.297843:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1786176930.297845:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176930.297846:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176930.297849:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176930.297850:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786176930.297853:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.297856:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176930.297858:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000040:1.0:1786176930.297859:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786176930.297860:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786176930.297863:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176930.297866:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786176930.297870:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786176930.297875:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176930.297877:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:3.0:1786176930.297880:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176930.297881:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.297886:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1786176930.297887:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1786176930.297890:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176930.297891:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176930.297892:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:1.0:1786176930.297895:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786176930.297897:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00100000:1.0:1786176930.297902:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315648:192.168.202.110@tcp:38:kworker.0
00000100:00080000:3.0:1786176930.297905:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176930.297906:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176930.297910:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1786176930.297911:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176930.297912:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176930.297917:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1786176930.297918:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:1.0:1786176930.297921:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:3.0:1786176930.297923:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176930.297924:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176930.297931:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176930.297932:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176930.297937:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00000001:3.0:1786176930.297937:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176930.297942:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000010:1.0:1786176930.297943:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176930.297948:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856315648, portal 10
00000008:00000001:3.0:1786176930.297949:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.297951:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786176930.297953:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.297954:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764293504 : -114547945258112 : ffff97d1ba69c380)
00000100:00000010:3.0:1786176930.297962:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ed80.
00000100:00000040:1.0:1786176930.297963:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786176930.297967:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176930.297973:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176930.297975:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.297977:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786176930.297978:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:1.0:1786176930.297982:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786176930.297982:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:1.0:1786176930.297988:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
02000000:00000001:3.0:1786176930.297988:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176930.297994:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856315648, offset 0
02000000:00000010:3.0:1786176930.297997:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000400:00000200:1.0:1786176930.298000:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176930.298005:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176930.298006:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176930.298013:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176930.298018:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:3.0:1786176930.298018:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176930.298023:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176930.298027:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176930.298033:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.298039:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176930.298040:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254637:254637:256:4294967295] 192.168.202.110@tcp LPNI seq info [254626:254626:8:4294967295]
00000100:00000001:3.0:1786176930.298044:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176930.298048:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1786176930.298053:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ccd6c6d2] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176930.298061:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786176930.298073:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176930.298074:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786176930.298080:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:1.0:1786176930.298081:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000100:00080000:3.0:1786176930.298087:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:1.0:1786176930.298091:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786176930.298091:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176930.298093:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:1.0:1786176930.298095:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:3.0:1786176930.298097:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176930.298102:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7153
00000800:00000200:1.1:1786176930.298105:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:3.0:1786176930.298109:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:1.1:1786176930.298110:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000400:00000200:3.0:1786176930.298116:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000100:1.0:1786176930.298119:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786176930.298120:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000010:1.0:1786176930.298125:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000100:00000040:3.0:1786176930.298126:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.298131:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176930.298132:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1786176930.298133:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.298139:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176930.298142:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176930.298144:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000400:00000200:1.0:1786176930.298145:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:3.0:1786176930.298146:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176930.298150:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000400:00000200:1.0:1786176930.298153:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000020:00000001:3.0:1786176930.298153:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786176930.298155:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000010:1.0:1786176930.298159:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000040:3.0:1786176930.298160:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.298164:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176930.298166:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:3.0:1786176930.298166:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176930.298170:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176930.298172:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000200:1.0:1786176930.298176:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176930.298177:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176930.298182:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176930.298187:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176930.298190:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176930.298193:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1786176930.298193:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.298197:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:1.0:1786176930.298200:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1786176930.298201:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176930.298206:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.298210:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176930.298216:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e680.
00000100:00000001:1.0:1786176930.298217:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.298220:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:3.0:1786176930.298220:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176930.298225:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176930.298226:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176930.298227:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176930.298232:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:1.0:1786176930.298233:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
02000000:00000001:3.0:1786176930.298234:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176930.298237:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176930.298241:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176930.298244:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301826183
02000000:00000010:3.0:1786176930.298244:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:1.0:1786176930.298250:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301829837
00000100:00000001:3.0:1786176930.298250:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.298252:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301826183
00000100:00000001:1.0:1786176930.298256:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301829837
00000100:00000001:1.0:1786176930.298259:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1786176930.298259:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176930.298265:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.298268:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176930.298275:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176930/real 1786176930]  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176930.298276:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176930.298280:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176930.298288:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1839131] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176930.298290:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176930.298292:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176930.298295:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176930.298298:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000100:00000001:3.0:1786176930.298301:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176930.298302:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176930.298307:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:3.0:1786176930.298308:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7163)
00000100:00000200:1.0:1786176930.298312:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176930.298319:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298324:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176930.298325:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298326:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176930.298327:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176930.298330:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298334:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176930.298336:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298358:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176930.298360:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298361:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176930.298362:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298363:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176930.298364:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176930.298366:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176930.298369:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176930.298371:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176930.298374:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176930.298379:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176930.298382:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298383:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176930.298385:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c380 x1872934856315648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298389:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176930.298390:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176930.298392:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176930.298394:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176930.298395:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176930.298396:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176930.298397:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176930.298398:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:1.0:1786176930.298400:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176930.298401:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176930.298402:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c380.
00000100:00000001:1.0:1786176930.298404:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176930.298405:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298406:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298409:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176930.298409:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.298411:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176930.298412:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176930.298413:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176930.298416:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298426:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176930.298427:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176930.298429:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176930.298430:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176930.298432:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176930.298434:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176930.298436:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176930.298438:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176930.298438:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176930.298440:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176930.298442:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176930.298443:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176930.298446:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176930.298448:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176930.298451:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176930.298453:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856315776, portal 4
00000100:00000001:1.0:1786176930.298455:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176930.298455:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:1.0:1786176930.298460:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298464:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176930.298466:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176930.298468:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176930.298469:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856315776, offset 0
00000400:00000200:1.0:1786176930.298472:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176930.298475:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176930.298477:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176930.298479:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176930.298481:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176930.298483:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176930.298485:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176930.298487:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254638:254638:256:4294967295] 192.168.202.110@tcp LPNI seq info [254627:254627:8:4294967295]
00000400:00000200:1.0:1786176930.298490:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176930.298494:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176930.298498:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176930.298500:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176930.298503:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176930.298505:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786176930.298508:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176930.298527:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1786176930.298529:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176930.298531:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176930.298533:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176930.298536:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176930.298538:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176930.298539:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176930.298541:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176930.298543:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298549:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176930.298550:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298555:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.298556:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176930.298558:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176930.298560:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176930.298563:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176930.298564:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301826059
00000100:00000001:1.0:1786176930.298566:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301829837
00000100:00000001:1.0:1786176930.298567:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301826059
00000100:00000001:1.0:1786176930.298569:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301829837
00000100:00000001:1.0:1786176930.298570:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176930.298572:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.298573:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176930.298575:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176930/real 1786176930]  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176930.298583:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176930.298585:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176930.298587:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176930.298589:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176930.298591:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176930.298593:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176930.298596:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176930.298606:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298613:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176930.298615:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298617:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176930.298618:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176930.298622:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298630:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176930.298634:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298639:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176930.298641:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298642:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176930.298643:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298644:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176930.298645:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176930.298647:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176930.298649:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176930.298650:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176930.298653:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176930.298659:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176930.298661:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176930.298664:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298667:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176930.298668:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176930.298670:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176930.298671:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176930.298672:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176930.298674:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315904:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176930.298676:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176930.298677:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176930.298679:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176930.298680:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176930.298683:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176930.298684:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176930.298686:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176930.298688:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786176930.298690:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786176930.298693:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856315904, portal 4
00000100:00000001:1.0:1786176930.298694:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176930.298695:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00000040:1.0:1786176930.298699:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298703:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176930.298705:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176930.298706:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000200:1.0:1786176930.298708:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856315904, offset 0
00000400:00000200:1.0:1786176930.298711:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176930.298713:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176930.298716:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176930.298718:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176930.298720:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176930.298722:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176930.298723:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176930.298725:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254639:254639:256:4294967295] 192.168.202.110@tcp LPNI seq info [254628:254628:8:4294967295]
00000400:00000200:1.0:1786176930.298728:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176930.298732:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176930.298735:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176930.298737:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000800:00000200:1.0:1786176930.298740:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176930.298742:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1786176930.298745:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176930.298747:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1786176930.298751:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176930.298753:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:1.0:1786176930.298756:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176930.298760:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176930.298763:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:1.0:1786176930.298765:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:1.0:1786176930.298768:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176930.298772:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298780:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176930.298784:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298791:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.298792:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176930.298796:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176930.298799:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176930.298802:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176930.298805:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301826067
00000100:00000001:1.0:1786176930.298808:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301829837
00000100:00000001:1.0:1786176930.298810:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301826067
00000100:00000001:1.0:1786176930.298813:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301829837
00000100:00000001:1.0:1786176930.298815:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176930.298817:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.298819:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176930.298824:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176930/real 1786176930]  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176930.298832:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176930.298834:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176930.298835:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176930.298838:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176930.298840:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176930.298842:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176930.298845:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176930.298853:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298858:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176930.298859:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298860:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176930.298861:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176930.298863:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298868:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176930.298870:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298874:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176930.298877:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298878:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176930.298878:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298880:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176930.298881:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176930.298883:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176930.298884:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176930.298886:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176930.298888:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176930.298893:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856315904:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176930.298896:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298897:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176930.298899:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856315776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298903:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176930.298905:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176930.298906:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176930.298908:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176930.298909:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176930.298910:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176930.298911:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176930.298912:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:1.0:1786176930.298914:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176930.298915:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176930.298916:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:1.0:1786176930.298918:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176930.298919:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298920:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176930.298922:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856315904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176930.298930:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176930.298931:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176930.298932:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176930.298934:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176930.298935:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176930.298935:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176930.298936:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176930.298937:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176930.298938:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176930.298939:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176930.298940:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:1.0:1786176930.298941:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176930.298942:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298943:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176930.298945:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176930.298946:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176930.298947:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176930.298948:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176930.872636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176930.872644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176930.872651:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176930.872652:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176930.872656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176930.872657:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176930.872711:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176930.872715:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176930.872718:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176931.320655:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176931.320664:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176931.320666:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176931.320671:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176931.320674:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176931.320678:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176931.320682:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176931.320684:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176931.320686:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176931.896674:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176931.896685:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176931.896693:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176931.896695:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176931.896698:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176931.896700:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176931.896808:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176931.896815:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176931.896821:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176932.344670:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176932.344679:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176932.344681:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176932.344687:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176932.344691:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176932.344695:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176932.344697:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176932.344700:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176932.344703:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176932.920604:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176932.920615:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176932.920624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176932.920626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176932.920630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176932.920634:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176932.920637:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176932.920642:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176932.920646:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176932.920650:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176932.920652:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176932.920654:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176932.920781:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176932.920788:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176932.920791:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176933.368603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176933.368612:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176933.368614:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176933.368619:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176933.368623:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176933.368626:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176933.368629:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176933.368632:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176933.368634:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176933.944611:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176933.944618:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176933.944623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176933.944624:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176933.944628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176933.944630:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176933.944715:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176933.944721:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176933.944724:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176934.136658:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301829837, at 4301829888
00000100:00000040:1.0:1786176934.136658:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301829837, at 4301829888
00000100:00000010:0.0:1786176934.136685:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1960.
00000100:00000010:1.0:1786176934.136689:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1cc0.
00000100:00000001:1.0:1786176934.392619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176934.392626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176934.392628:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176934.392632:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176934.392635:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176934.392638:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176934.392640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176934.392642:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176934.392644:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176934.968580:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176934.968586:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176934.968589:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176934.968590:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176934.968593:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176934.968594:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176934.968666:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176934.968670:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176934.968673:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176935.416597:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1786176935.416597:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176935.416603:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176935.416606:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176935.416607:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:1.0:1786176935.416612:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176935.416612:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786176935.416617:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1786176935.416617:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176935.416622:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176935.416623:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176935.416625:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176935.416628:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176935.416630:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176935.416634:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7158
00000400:00000200:2.0:1786176935.416645:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176935.416653:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176935.416657:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176935.416660:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.416663:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176935.416666:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.416669:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176935.416673:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176935.416675:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176935.416679:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176935.416682:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176935.416684:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176935.416686:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176935.416689:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.416690:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176935.416692:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.416693:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176935.416695:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176935.416697:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176935.416698:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176935.416700:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176935.416703:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176935.416705:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176935.416706:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176935.416708:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176935.416716:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000020:00000040:2.0:1786176935.416719:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176935.416722:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176935.416724:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176935.416726:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176935.416728:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176935.416731:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176935.416738:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786176935.416743:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176935.416750:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176935.416758:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176935.416762:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176935.416765:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176935.416783:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176935.416785:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176935.416788:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176935.416789:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176935.416790:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176935.416792:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176935.416794:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7158
00000400:00000200:2.0:1786176935.416797:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176935.416799:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176935.416801:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176935.416803:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.416805:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176935.416806:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.416808:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176935.416810:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176935.416811:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176935.416812:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176935.416814:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176935.416815:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176935.416817:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1786176935.416817:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176935.416822:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176935.416823:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176935.416824:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176935.416829:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786176935.416829:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.416833:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176935.416835:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176935.416839:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:1.0:1786176935.416842:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176935.416844:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176935.416849:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176935.416854:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176935.416858:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1786176935.416858:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176935.416862:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176935.416863:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1786176935.416865:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786176935.416869:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:2.0:1786176935.416869:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176935.416874:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176935.416876:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:2.0:1786176935.416876:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000020:00000040:2.0:1786176935.416881:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786176935.416882:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176935.416884:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176935.416886:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176935.416889:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:1.0:1786176935.416891:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856316032:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1786176935.416892:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176935.416894:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176935.416899:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:1.0:1786176935.416900:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1786176935.416903:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176935.416906:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176935.416910:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176935.416910:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786176935.416915:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786176935.416921:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176935.416923:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176935.416924:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
00000100:00000040:2.0:1786176935.416927:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786176935.416929:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176935.416938:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176935.416940:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786176935.416941:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176935.416946:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176935.416947:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:1.0:1786176935.416948:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000100:00080000:2.0:1786176935.416949:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176935.416953:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176935.416956:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7158
00000400:00000200:2.0:1786176935.416959:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176935.416960:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000400:00000200:2.0:1786176935.416963:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176935.416966:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176935.416968:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.416969:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176935.416971:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:1.0:1786176935.416972:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856316032, portal 25
00000100:00000001:2.0:1786176935.416973:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176935.416979:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176935.416980:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000020:00000040:2.0:1786176935.416980:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176935.416983:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176935.416985:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000020:00000040:2.0:1786176935.416986:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176935.416989:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176935.416991:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:1.0:1786176935.416995:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176935.416996:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.417001:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176935.417006:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.417011:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176935.417013:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:2.0:1786176935.417014:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000040:1.0:1786176935.417018:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176935.417020:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:1.0:1786176935.417022:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cee0.
00000100:00000001:2.0:1786176935.417023:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176935.417027:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:1.0:1786176935.417028:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856316032, offset 0
02000000:00000001:2.0:1786176935.417029:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176935.417032:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176935.417033:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176935.417035:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1786176935.417037:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000008:00000001:2.0:1786176935.417039:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176935.417041:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176935.417045:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000400:00000200:1.0:1786176935.417048:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786176935.417049:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176935.417052:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176935.417053:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176935.417056:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1786176935.417057:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176935.417060:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176935.417063:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176935.417065:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000010:2.0:1786176935.417066:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000400:00000200:1.0:1786176935.417071:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1786176935.417071:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176935.417077:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00100000:2.0:1786176935.417079:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176935.417082:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176935.417090:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254640:254640:256:4294967295] 192.168.202.110@tcp LPNI seq info [254629:254629:8:4294967295]
00000100:00000001:2.0:1786176935.417098:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176935.417100:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176935.417103:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176935.417107:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176935.417115:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176935.417127:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176935.417127:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176935.417131:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:1.0:1786176935.417134:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a700.
00000100:00080000:2.0:1786176935.417134:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176935.417138:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176935.417140:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176935.417142:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.0:1786176935.417144:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786176935.417147:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7158
00000400:00000200:2.0:1786176935.417154:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:1.1:1786176935.417154:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:2.0:1786176935.417160:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176935.417162:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176935.417165:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.417166:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176935.417167:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.417168:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176935.417170:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176935.417170:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176935.417171:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176935.417173:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:1.0:1786176935.417174:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176935.417175:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176935.417180:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:1.0:1786176935.417184:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786176935.417186:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176935.417188:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301834957
00000100:00000001:2.0:1786176935.417189:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176935.417193:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000010:1.0:1786176935.417196:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1cc0.
00000100:00000001:2.0:1786176935.417196:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176935.417201:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176935.417204:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176935.417205:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786176935.417206:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176935.417209:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176935.417212:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176935.417213:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417214:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301834957
00000100:00000001:2.0:1786176935.417215:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176935.417219:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000010:1.0:1786176935.417220:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1720.
00000008:00000001:2.0:1786176935.417225:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417226:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786176935.417227:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417232:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:2.0:1786176935.417233:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000100:00000001:1.0:1786176935.417237:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786176935.417238:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176935.417242:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176935.417242:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176935.417245:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417246:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176935.417247:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176935.417251:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176935.417252:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417255:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000010:2.0:1786176935.417256:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:1.0:1786176935.417259:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1786176935.417261:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417262:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176935.417265:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176935.417268:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00100000:2.0:1786176935.417268:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1786176935.417275:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176935.417279:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176935.417283:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417289:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176935.417304:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176935.417305:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786176935.417307:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176935.417310:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176935.417315:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176935.417321:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856316160:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786176935.417322:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417326:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176935.417328:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176935.417329:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176935.417331:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:2.0:1786176935.417335:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000010:1.0:1786176935.417336:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176935.417340:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176935.417342:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176935.417345:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1786176935.417346:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786176935.417348:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1786176935.417351:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:2.0:1786176935.417352:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:1.0:1786176935.417358:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8ca18.
00000100:00080000:2.0:1786176935.417358:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7158
00000100:00000200:1.0:1786176935.417363:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856316160, portal 10
00000400:00000200:2.0:1786176935.417365:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176935.417367:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176935.417369:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178944 : -114548255372672 : ffff97d1a7edca80)
00000400:00000200:2.0:1786176935.417371:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176935.417374:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176935.417379:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176935.417380:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176935.417382:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176935.417387:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.417392:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176935.417393:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:2.0:1786176935.417397:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000040:1.0:1786176935.417398:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:2.0:1786176935.417400:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000400:00000010:1.0:1786176935.417402:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c5d8.
00000020:00000001:2.0:1786176935.417403:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176935.417407:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000200:1.0:1786176935.417408:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856316160, offset 0
00000020:00000001:2.0:1786176935.417411:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:1.0:1786176935.417415:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:2.0:1786176935.417415:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786176935.417421:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176935.417422:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176935.417424:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176935.417428:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1786176935.417429:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176935.417435:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176935.417437:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:2.0:1786176935.417439:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000400:00000200:1.0:1786176935.417443:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:2.0:1786176935.417446:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1786176935.417450:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1786176935.417451:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176935.417454:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:2.0:1786176935.417455:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176935.417459:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786176935.417460:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254641:254641:255:4294967295] 192.168.202.110@tcp LPNI seq info [254630:254630:7:4294967295]
02000000:00000001:2.0:1786176935.417465:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176935.417469:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176935.417470:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176935.417474:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176935.417480:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176935.417481:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786176935.417483:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176935.417487:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000010:2.0:1786176935.417489:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000800:00000010:1.0:1786176935.417492:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ab00.
00000020:00000040:2.0:1786176935.417494:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000800:00000200:1.0:1786176935.417499:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1786176935.417501:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:1.1:1786176935.417505:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1786176935.417506:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417527:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176935.417532:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176935.417535:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176935.417537:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301831178
00000100:00000001:1.0:1786176935.417541:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301834957
00000100:00000001:1.0:1786176935.417544:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301831178
00000100:00000001:1.0:1786176935.417546:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301834957
00000100:00000001:1.0:1786176935.417549:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176935.417552:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417554:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786176935.417559:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:1.0:1786176935.417563:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176935.417563:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176935.417566:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417576:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000010:2.0:1786176935.417576:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:1.0:1786176935.417579:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176935.417584:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1786176935.417585:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176935.417587:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176935.417592:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176935.417596:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:1.0:1786176935.417600:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856316288:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176935.417610:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176935.417613:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176935.417616:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176935.417617:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1786176935.417621:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786176935.417622:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176935.417627:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
00000100:00000040:2.0:1786176935.417631:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786176935.417632:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176935.417641:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176935.417646:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d1b8.
00000100:00000001:2.0:1786176935.417652:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176935.417654:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c4c8.
00000100:00000200:1.0:1786176935.417659:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856316288, portal 10
00000100:00000040:2.0:1786176935.417661:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7168)
00000100:00000001:1.0:1786176935.417662:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176935.417666:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000040:1.0:1786176935.417674:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176935.417681:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176935.417684:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176935.417687:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c330.
00000100:00000200:1.0:1786176935.417689:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856316288, offset 0
00000400:00000200:1.0:1786176935.417693:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176935.417697:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176935.417701:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176935.417704:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176935.417708:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176935.417710:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176935.417713:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176935.417716:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254642:254642:254:4294967295] 192.168.202.110@tcp LPNI seq info [254631:254631:6:4294967295]
00000400:00000200:1.0:1786176935.417719:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176935.417726:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176935.417730:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176935.417733:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881af00.
00000800:00000200:1.0:1786176935.417737:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176935.417741:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1786176935.417744:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176935.417747:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176935.417750:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176935.417751:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301831303
00000100:00000001:1.0:1786176935.417754:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301834957
00000100:00000001:1.0:1786176935.417756:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301831303
00000100:00000001:1.0:1786176935.417759:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301834957
00000100:00000001:1.0:1786176935.417761:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176935.417763:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417766:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417768:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176935.417771:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176935.417773:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1786176935.417775:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176935.417776:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176935.417778:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176935.417781:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176935.417788:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176935.417790:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176935.417792:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176935.417794:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176935.417796:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176935.417799:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856316416:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176935.417802:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176935.417804:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176935.417806:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176935.417808:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176935.417811:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176935.417813:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176935.417818:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176935.417821:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8db58.
00000400:00000010:1.0:1786176935.417825:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176935.417833:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856316416, portal 4
00000100:00000001:1.0:1786176935.417837:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176935.417838:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454188800 : -114548255362816 : ffff97d1a7edf100)
00000100:00000040:1.0:1786176935.417845:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176935.417853:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176935.417855:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176935.417859:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cd48.
00000100:00000200:1.0:1786176935.417861:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856316416, offset 0
00000400:00000200:1.0:1786176935.417866:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176935.417869:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176935.417872:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176935.417875:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176935.417878:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176935.417880:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176935.417882:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176935.417886:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254643:254643:253:4294967295] 192.168.202.110@tcp LPNI seq info [254632:254632:5:4294967295]
00000400:00000200:1.0:1786176935.417891:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176935.417897:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176935.417903:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176935.417907:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ae00.
00000800:00000200:1.0:1786176935.417911:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176935.417926:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1786176935.417930:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176935.417933:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176935.417936:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176935.417939:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301831178
00000100:00000001:1.0:1786176935.417941:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301834957
00000100:00000001:1.0:1786176935.417943:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301831178
00000100:00000001:1.0:1786176935.417945:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301834957
00000100:00000001:1.0:1786176935.417948:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176935.417951:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.417953:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176935.417957:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176935.417964:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176935.417965:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176935.417967:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176935.417969:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176935.417972:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176935.417975:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856316544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176935.417978:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176935.417980:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176935.417981:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176935.417983:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176935.417987:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176935.417989:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176935.417992:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176935.417995:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176935.417999:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176935.418003:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856316544, portal 4
00000100:00000001:1.0:1786176935.418005:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176935.418006:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00000040:1.0:1786176935.418012:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176935.418019:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176935.418022:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176935.418024:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cdd0.
00000100:00000200:1.0:1786176935.418026:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856316544, offset 0
00000400:00000200:1.0:1786176935.418031:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176935.418036:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176935.418040:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176935.418043:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176935.418045:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176935.418054:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176935.418057:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176935.418060:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254644:254644:252:4294967295] 192.168.202.110@tcp LPNI seq info [254633:254633:4:4294967295]
00000400:00000200:1.0:1786176935.418064:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176935.418071:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176935.418076:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176935.418079:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a600.
00000800:00000200:1.0:1786176935.418083:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176935.418087:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1786176935.418089:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176935.418093:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176935.418096:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176935.418098:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301831186
00000100:00000001:1.0:1786176935.418100:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301834957
00000100:00000001:1.0:1786176935.418102:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301831186
00000100:00000001:1.0:1786176935.418104:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301834957
00000100:00000001:1.0:1786176935.418107:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176935.418109:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.418112:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.418114:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176935.418118:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176935.418119:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1786176935.418121:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176935.418123:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176935.418125:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176935.418127:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176935.992612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176935.992617:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176935.992621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176935.992622:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176935.992624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176935.992625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176935.992687:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176935.992691:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176935.992693:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176936.440627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176936.440634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176936.440635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176936.440640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176936.440643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176936.440647:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176936.440649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176936.440651:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176936.440653:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176937.016600:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176937.016606:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176937.016610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176937.016611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176937.016613:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176937.016614:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176937.016665:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176937.016670:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176937.016674:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176937.464628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176937.464634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176937.464636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176937.464642:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176937.464646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176937.464651:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176937.464654:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176937.464656:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176937.464660:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176938.040650:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176938.040658:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176938.040663:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176938.040665:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176938.040668:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176938.040676:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176938.040776:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176938.040783:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176938.040789:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176938.488640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176938.488650:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176938.488653:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176938.488659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176938.488664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176938.488667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.488671:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176938.488674:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.488677:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:0.0:1786176938.488775:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:0.0:1786176938.488817:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:0.0:1786176938.488828:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176938.488832:0:2380:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:0.0:1786176938.488837:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176938.488841:0:2380:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:0.0:1786176938.488847:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1786176938.488851:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a700.
00000400:00000200:0.0:1786176938.488857:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1786176938.488861:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1786176938.488868:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cee0
00000400:00000010:0.0:1786176938.488871:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cee0.
00000100:00000001:0.0:1786176938.488877:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176938.488886:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176938.488912:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176938.488918:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176938.488926:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176938.488928:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1786176938.488937:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1786176938.488941:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ab00.
00000400:00000200:0.0:1786176938.488943:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1786176938.488947:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1786176938.488950:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c5d8
00000400:00000010:0.0:1786176938.488952:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c5d8.
00000100:00000001:0.0:1786176938.488954:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176938.488959:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176938.488967:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176938.488971:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176938.488978:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176938.488980:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176938.488981:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176938.488986:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000800:00000100:0.0:1786176938.488991:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000100:00000001:1.0:1786176938.488994:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000800:00000010:0.0:1786176938.488995:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881af00.
00000400:00000200:0.0:1786176938.489001:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1786176938.489007:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000100:00100000:1.0:1786176938.489007:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176935/real 1786176938]  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786176938.489014:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c330
00000400:00000010:0.0:1786176938.489018:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c330.
00000100:00000001:0.0:1786176938.489023:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176938.489031:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176938.489033:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176938.489038:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176938.489042:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786176938.489047:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8ca18
00000400:00000010:1.0:1786176938.489052:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8ca18.
00000100:00000001:1.0:1786176938.489059:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1786176938.489064:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000200:1.0:1786176938.489069:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176938.489074:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176938.489099:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176938.489102:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176938.489103:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1786176938.489114:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1786176938.489120:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ae00.
00000100:00000001:1.0:1786176938.489123:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000400:00000200:0.0:1786176938.489125:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:1.0:1786176938.489128:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1786176938.489133:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000100:00000001:1.0:1786176938.489134:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000400:00000200:0.0:1786176938.489138:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cd48
00000100:00000001:1.0:1786176938.489139:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000010:0.0:1786176938.489142:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cd48.
00000100:00000001:0.0:1786176938.489146:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:1.0:1786176938.489149:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786176938.489155:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489172:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176938.489176:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176938.489184:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176938.489185:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176938.489219:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176938.489222:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786176938.489223:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176938.489229:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000800:00000100:0.0:1786176938.489234:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000100:00000001:1.0:1786176938.489235:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176938.489239:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000800:00000010:0.0:1786176938.489240:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a600.
00000100:00000001:1.0:1786176938.489245:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000400:00000200:0.0:1786176938.489246:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:1.0:1786176938.489251:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000400:00000200:0.0:1786176938.489254:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000100:00080000:1.0:1786176938.489260:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000400:00000200:0.0:1786176938.489278:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cdd0
00000400:00000010:0.0:1786176938.489281:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cdd0.
00000100:00000001:0.0:1786176938.489284:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786176938.489289:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176938.489297:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176938.489303:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1786176938.489309:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176938.489317:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176938.489317:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1786176938.489323:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:1.0:1786176938.489328:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176938.489341:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856316160:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176938.489347:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176938.489351:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176935/real 1786176938]  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176938.489361:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176938.489363:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176938.489366:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d1b8.
00000400:00000200:1.0:1786176938.489370:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c4c8
00000400:00000010:1.0:1786176938.489372:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c4c8.
00000100:00000001:1.0:1786176938.489375:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176938.489378:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176938.489388:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489395:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176938.489397:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489399:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176938.489400:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176938.489404:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489411:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176938.489415:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489422:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176938.489425:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489426:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176938.489428:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489429:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176938.489432:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176938.489435:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176938.489438:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176938.489442:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176938.489446:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176938.489454:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856316288:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176938.489459:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176938.489463:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176935/real 1786176938]  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176938.489485:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176938.489486:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176938.489489:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000400:00000200:1.0:1786176938.489492:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786176938.489494:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786176938.489496:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176938.489500:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176938.489507:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489538:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176938.489541:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489543:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176938.489545:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176938.489551:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489558:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176938.489562:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489568:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176938.489571:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489572:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176938.489574:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489575:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176938.489577:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176938.489580:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176938.489584:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176938.489586:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176938.489590:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176938.489599:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856316032:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176938.489603:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176938.489605:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176938.489608:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edca80 x1872934856316160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489614:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176938.489617:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176938.489620:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176938.489625:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176938.489627:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176938.489629:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176938.489631:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176938.489634:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:1.0:1786176938.489638:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176938.489640:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176938.489642:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edca80.
00000100:00000001:1.0:1786176938.489645:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176938.489647:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489648:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176938.489651:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856316288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489657:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176938.489658:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176938.489659:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176938.489662:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176938.489663:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176938.489664:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176938.489666:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176938.489667:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3400.
02000000:00000001:1.0:1786176938.489669:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176938.489670:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176938.489672:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:1.0:1786176938.489674:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176938.489675:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489677:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176938.489680:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856316032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489686:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176938.489687:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176938.489688:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176938.489691:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176938.489692:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176938.489694:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176938.489695:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176938.489697:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:1.0:1786176938.489700:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176938.489701:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176938.489702:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
00000100:00000001:1.0:1786176938.489704:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176938.489705:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489709:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176938.489711:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176938.489713:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176938.489714:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176938.489718:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176935/real 1786176938]  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176938.489733:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176938.489735:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176938.489737:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8db58.
00000400:00000200:1.0:1786176938.489740:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176938.489742:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176938.489744:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176938.489748:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176938.489757:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489770:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176938.489777:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489780:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176938.489781:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176938.489786:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489794:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176938.489798:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489805:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176938.489808:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489810:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176938.489811:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489813:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176938.489815:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176938.489818:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176938.489822:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176938.489825:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176938.489828:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176938.489837:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856316416:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176938.489843:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176938.489847:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176935/real 1786176938]  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176938.489856:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176938.489858:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176938.489861:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176938.489865:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176938.489866:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176938.489869:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176938.489872:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176938.489882:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489890:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176938.489892:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489894:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176938.489896:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176938.489901:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489909:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176938.489913:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489921:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176938.489923:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489925:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176938.489927:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489929:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176938.489931:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176938.489934:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176938.489937:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176938.489940:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176938.489944:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176938.489953:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856316544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176938.489963:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.489965:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176938.489969:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf100 x1872934856316416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.489976:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176938.489978:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176938.489980:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176938.489986:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176938.489987:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176938.489988:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176938.489991:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176938.489992:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0800.
02000000:00000001:1.0:1786176938.489997:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176938.489998:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176938.490000:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf100.
00000100:00000001:1.0:1786176938.490003:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176938.490005:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.490007:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176938.490010:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856316544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176938.490017:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176938.490018:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176938.490019:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176938.490021:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176938.490022:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176938.490023:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176938.490025:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176938.490026:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:1.0:1786176938.490028:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176938.490029:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176938.490030:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:1.0:1786176938.490031:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176938.490032:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.490034:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176938.490041:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176938.490043:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176938.490045:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176938.490046:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176939.064598:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176939.064604:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176939.064609:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176939.064610:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176939.064612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176939.064613:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176939.064670:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176939.064673:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176939.064676:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176939.256599:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301834957, at 4301835008
00000100:00000040:1.0:1786176939.256599:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301834957, at 4301835008
00000100:00000010:0.0:1786176939.256613:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1cc0.
00000100:00000010:1.0:1786176939.256619:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1720.
00000100:00000001:1.0:1786176939.512668:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176939.512673:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176939.512675:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176939.512680:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176939.512683:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176939.512686:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176939.512688:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176939.512690:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176939.512693:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176940.088590:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176940.088599:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176940.088602:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176940.088605:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176940.088610:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176940.088613:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176940.088615:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176940.088617:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176940.088621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176940.088623:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176940.088624:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176940.088626:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176940.088627:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176940.088629:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176940.088631:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176940.088632:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176940.088636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176940.088638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176940.088683:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176940.088687:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176940.088689:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786176940.536618:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176940.536626:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786176940.536628:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176940.536632:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176940.536636:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176940.536642:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7163
00000400:00000200:0.0:1786176940.536649:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176940.536655:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176940.536657:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176940.536660:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176940.536661:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786176940.536663:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176940.536667:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176940.536668:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786176940.536669:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176940.536673:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176940.536674:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176940.536677:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000100:00100000:1.0:1786176940.536680:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000020:00000001:0.0:1786176940.536681:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176940.536684:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786176940.536687:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000001:1.0:1786176940.536687:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1786176940.536691:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000001:1.0:1786176940.536691:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176940.536695:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176940.536697:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176940.536698:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176940.536701:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176940.536701:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176940.536706:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176940.536707:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786176940.536711:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786176940.536716:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176940.536719:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176940.536722:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176940.536725:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176940.536728:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.536731:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786176940.536734:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176940.536741:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418a80.
00000020:00000040:0.0:1786176940.536745:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786176940.536749:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176940.536751:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.536754:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176940.536757:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176940.536762:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176940.536770:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
00000100:00000001:0.0:1786176940.536777:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176940.536786:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176940.536796:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176940.536800:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176940.536811:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000606c34cb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176940.536835:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176940.536838:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176940.536841:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786176940.536842:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176940.536843:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176940.536846:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176940.536848:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7163
00000400:00000200:0.0:1786176940.536851:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176940.536855:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176940.536857:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176940.536860:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176940.536861:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176940.536864:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176940.536866:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176940.536867:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176940.536868:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786176940.536870:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176940.536870:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1786176940.536874:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:1.0:1786176940.536875:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1786176940.536876:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176940.536879:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1786176940.536880:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786176940.536885:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176940.536889:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786176940.536890:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176940.536895:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176940.536900:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176940.536905:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786176940.536911:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1786176940.536912:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176940.536915:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.536917:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176940.536921:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786176940.536923:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176940.536928:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176940.536931:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176940.536936:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856316672:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176940.536941:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176940.536942:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176940.536946:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176940.536947:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176940.536951:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.536952:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786176940.536957:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:0.0:1786176940.536958:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786176940.536963:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176940.536964:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176940.536969:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.536970:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176940.536977:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1786176940.536978:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418e00.
00000020:00000040:0.0:1786176940.536983:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000400:00000010:1.0:1786176940.536984:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
02000000:00000001:0.0:1786176940.536987:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176940.536990:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.536993:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:1.0:1786176940.536994:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
02000000:00000001:0.0:1786176940.536996:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176940.537000:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176940.537003:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856316672, portal 25
02000000:00000010:0.0:1786176940.537006:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4c00.
00000100:00000001:1.0:1786176940.537008:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176940.537013:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761657472 : -114547947894144 : ffff97d1ba418a80)
00000100:00000001:0.0:1786176940.537015:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176940.537021:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176940.537026:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176940.537049:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176940.537053:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.537060:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176940.537062:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003428892b] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786176940.537068:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176940.537074:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000001:0.0:1786176940.537079:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176940.537080:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856316672, offset 0
00000100:00080000:0.0:1786176940.537085:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176940.537090:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786176940.537093:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1786176940.537095:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786176940.537096:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176940.537101:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176940.537105:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7163
00000400:00000200:1.0:1786176940.537108:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176940.537119:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176940.537125:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176940.537130:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176940.537134:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176940.537137:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176940.537141:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254645:254645:256:4294967295] 192.168.202.110@tcp LPNI seq info [254634:254634:8:4294967295]
00000400:00000200:1.0:1786176940.537147:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176940.537175:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176940.537184:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176940.537188:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000010:1.0:1786176940.537193:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000400:00000200:0.0:1786176940.537195:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176940.537200:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:1.0:1786176940.537206:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786176940.537207:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176940.537212:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1786176940.537215:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:0.0:1786176940.537218:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176940.537223:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:1.1:1786176940.537224:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:0.0:1786176940.537229:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000200:1.1:1786176940.537230:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000020:00000040:0.0:1786176940.537233:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000800:00000100:1.0:1786176940.537238:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:0.0:1786176940.537239:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000010:1.0:1786176940.537257:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176940.537264:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176940.537269:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:0.0:1786176940.537278:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:1.0:1786176940.537280:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176940.537285:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000020:00000001:0.0:1786176940.537287:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:1.0:1786176940.537291:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:0.0:1786176940.537292:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786176940.537301:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:1.0:1786176940.537301:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176940.537306:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176940.537310:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176940.537316:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176940.537322:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786176940.537325:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176940.537327:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176940.537329:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176940.537332:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176940.537337:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.537339:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786176940.537340:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176940.537345:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000040:1.0:1786176940.537349:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1786176940.537350:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.537354:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176940.537367:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419180.
00000100:00000001:1.0:1786176940.537369:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786176940.537372:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786176940.537373:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:0.0:1786176940.537378:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176940.537381:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.537383:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786176940.537383:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176940.537386:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176940.537390:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176940.537392:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1786176940.537396:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000001:1.0:1786176940.537398:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301840077
00000100:00000001:0.0:1786176940.537403:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786176940.537406:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1720.
00000100:00100000:0.0:1786176940.537412:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176940.537416:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176940.537426:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176940.537428:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301840077
00000100:00000001:0.0:1786176940.537429:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786176940.537434:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1d20.
00000100:00000040:0.0:1786176940.537437:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005a89d8ab] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.537440:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1786176940.537446:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176940.537454:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.537454:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176940.537460:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176940.537461:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00080000:0.0:1786176940.537467:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786176940.537471:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:1.0:1786176940.537471:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176940/real 1786176940]  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786176940.537476:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176940.537482:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176940.537488:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7163
00000400:00000001:1.0:1786176940.537489:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176940.537492:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000200:0.0:1786176940.537495:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176940.537497:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:0.0:1786176940.537502:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1786176940.537504:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000100:00080000:0.0:1786176940.537507:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176940.537550:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176940.537553:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176940.537555:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:1.0:1786176940.537556:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:0.0:1786176940.537562:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176940.537563:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:0.0:1786176940.537568:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176940.537571:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000200:1.0:1786176940.537573:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786176940.537575:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176940.537580:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786176940.537586:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786176940.537592:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000200:1.0:1786176940.537593:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176940.537602:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176940.537607:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176940.537608:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:0.0:1786176940.537611:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176940.537613:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176940.537617:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176940.537618:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:0.0:1786176940.537623:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:1.0:1786176940.537624:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786176940.537631:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:1.0:1786176940.537634:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176940.537636:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176940.537641:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176940.537647:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176940.537650:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786176940.537652:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.537657:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:1.0:1786176940.537660:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1786176940.537662:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786176940.537670:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.537674:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.537677:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000010:0.0:1786176940.537681:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b100.
00000100:00000001:1.0:1786176940.537683:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1786176940.537685:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176940.537688:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:0.0:1786176940.537691:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176940.537692:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786176940.537695:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.537698:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176940.537700:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786176940.537702:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:0.0:1786176940.537703:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176940.537707:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786176940.537709:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000010:0.0:1786176940.537715:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000100:00080000:1.0:1786176940.537716:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786176940.537722:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.537722:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00100000:0.0:1786176940.537734:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1786176940.537741:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176940.537747:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176940.537752:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176940.537762:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000fa632fb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176940.537763:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856316672:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176940.537773:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.537778:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786176940.537780:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176940.537785:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418a80 x1872934856316672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786176940.537787:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176940.537796:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786176940.537800:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786176940.537804:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00080000:0.0:1786176940.537805:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176940.537810:0:62275:0:(import.c:485:import_select_connection()) Process entered
02000000:00000001:1.0:1786176940.537810:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176940.537815:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176940.537821:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176940.537823:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176940.537825:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176940.537827:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176940.537830:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
00000100:00080000:0.0:1786176940.537831:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7163
02000000:00000001:1.0:1786176940.537835:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000400:00000200:0.0:1786176940.537840:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1786176940.537840:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176940.537844:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418a80.
00000400:00000200:0.0:1786176940.537846:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176940.537851:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00080000:0.0:1786176940.537852:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176940.537855:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176940.537859:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176940.537861:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176940.537863:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176940.537867:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176940.537871:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176940.537871:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000020:00000001:0.0:1786176940.537876:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176940.537876:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786176940.537880:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:1.0:1786176940.537882:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1786176940.537884:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176940.537886:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1786176940.537888:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:1.0:1786176940.537889:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1786176940.537894:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:1.0:1786176940.537898:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786176940.537899:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786176940.537907:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176940.537913:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176940.537917:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1786176940.537918:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176940.537920:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.537922:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:1.0:1786176940.537925:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1786176940.537927:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
02000000:00000001:1.0:1786176940.537928:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1786176940.537935:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:1.0:1786176940.537935:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.537941:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176940.537944:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856316800:192.168.202.110@tcp:38:kworker.0
02000000:00000001:0.0:1786176940.537945:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176940.537950:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176940.537951:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176940.537956:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176940.537956:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1786176940.537959:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786176940.537961:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1786176940.537963:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000001:1.0:1786176940.537965:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:0.0:1786176940.537969:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.537972:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176940.537972:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176940.537978:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176940.537980:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b480.
00000020:00000040:0.0:1786176940.537984:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000400:00000200:1.0:1786176940.537985:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786176940.537990:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:1.0:1786176940.537993:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
02000000:00000001:0.0:1786176940.537994:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176940.537999:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176940.538001:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176940.538003:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176940.538006:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
02000000:00000010:0.0:1786176940.538008:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7800.
00000100:00000200:1.0:1786176940.538013:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856316800, portal 10
00000100:00000001:1.0:1786176940.538019:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176940.538020:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.538021:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761658368 : -114547947893248 : ffff97d1ba418e00)
00000100:00100000:0.0:1786176940.538030:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1786176940.538035:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538062:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1786176940.538066:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1786176940.538070:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786176940.538072:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176940.538076:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000040:0.0:1786176940.538082:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001215e9e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176940.538083:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856316800, offset 0
00000400:00000200:1.0:1786176940.538097:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786176940.538100:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176940.538106:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786176940.538108:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7173)
00000400:00000200:1.0:1786176940.538113:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176940.538118:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176940.538122:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176940.538125:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176940.538127:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176940.538131:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254646:254646:256:4294967295] 192.168.202.110@tcp LPNI seq info [254635:254635:8:4294967295]
00000400:00000200:1.0:1786176940.538136:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176940.538144:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176940.538148:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176940.538153:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176940.538157:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176940.538160:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786176940.538164:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176940.538167:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1786176940.538170:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176940.538172:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176940.538175:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176940.538179:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176940.538183:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176940.538185:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176940.538188:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176940.538192:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538201:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176940.538205:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538211:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.538213:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176940.538217:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176940.538222:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176940.538226:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176940.538228:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301836298
00000100:00000001:1.0:1786176940.538231:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301840077
00000100:00000001:1.0:1786176940.538233:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301836298
00000100:00000001:1.0:1786176940.538235:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301840077
00000100:00000001:1.0:1786176940.538237:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176940.538239:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.538258:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176940.538263:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176940/real 1786176940]  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176940.538271:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176940.538273:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176940.538275:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176940.538278:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176940.538280:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176940.538282:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176940.538285:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176940.538295:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538302:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176940.538304:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.538306:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176940.538307:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176940.538311:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538319:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176940.538323:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538332:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176940.538334:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.538336:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176940.538338:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.538340:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176940.538342:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176940.538344:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176940.538348:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176940.538350:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176940.538354:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176940.538362:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856316800:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176940.538367:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176940.538370:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538377:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176940.538379:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176940.538383:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176940.538385:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176940.538386:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176940.538390:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856316928:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176940.538393:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176940.538395:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176940.538397:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176940.538398:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176940.538402:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176940.538404:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176940.538408:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176940.538410:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176940.538413:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176940.538422:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856316928, portal 10
00000100:00000001:1.0:1786176940.538425:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176940.538426:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761659264 : -114547947892352 : ffff97d1ba419180)
00000100:00000040:1.0:1786176940.538432:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538439:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176940.538441:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176940.538443:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176940.538445:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856316928, offset 0
00000400:00000200:1.0:1786176940.538450:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176940.538454:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176940.538458:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176940.538460:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176940.538463:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176940.538466:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176940.538468:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176940.538471:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254647:254647:256:4294967295] 192.168.202.110@tcp LPNI seq info [254636:254636:8:4294967295]
00000400:00000200:1.0:1786176940.538475:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176940.538481:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176940.538486:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176940.538489:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176940.538493:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176940.538496:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786176940.538500:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176940.538503:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1786176940.538506:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176940.538536:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176940.538540:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176940.538543:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176940.538545:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176940.538547:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176940.538549:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176940.538552:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538558:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176940.538560:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538564:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.538565:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176940.538568:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176940.538570:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176940.538573:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176940.538574:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301836424
00000100:00000001:1.0:1786176940.538576:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301840077
00000100:00000001:1.0:1786176940.538577:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301836424
00000100:00000001:1.0:1786176940.538579:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301840077
00000100:00000001:1.0:1786176940.538581:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176940.538582:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.538583:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176940.538585:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176940/real 1786176940]  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176940.538593:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176940.538594:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176940.538596:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176940.538599:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176940.538601:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176940.538603:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176940.538607:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176940.538614:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538622:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176940.538623:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.538625:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176940.538627:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176940.538630:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538637:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176940.538642:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538650:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176940.538653:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.538654:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176940.538655:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.538656:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176940.538658:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176940.538659:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176940.538662:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176940.538664:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176940.538668:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176940.538676:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856316928:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176940.538680:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176940.538684:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538691:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176940.538692:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176940.538695:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176940.538697:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176940.538699:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176940.538702:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856317056:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176940.538706:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176940.538708:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176940.538710:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176940.538712:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176940.538715:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176940.538718:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176940.538721:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176940.538724:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176940.538728:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176940.538732:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856317056, portal 4
00000100:00000001:1.0:1786176940.538734:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176940.538735:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761667328 : -114547947884288 : ffff97d1ba41b100)
00000100:00000040:1.0:1786176940.538741:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538748:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176940.538750:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176940.538752:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176940.538755:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856317056, offset 0
00000400:00000200:1.0:1786176940.538760:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176940.538764:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176940.538768:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176940.538772:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176940.538774:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176940.538776:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176940.538779:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176940.538782:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254648:254648:256:4294967295] 192.168.202.110@tcp LPNI seq info [254637:254637:8:4294967295]
00000400:00000200:1.0:1786176940.538786:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176940.538792:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176940.538797:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176940.538800:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176940.538804:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176940.538807:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786176940.538811:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176940.538814:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1786176940.538818:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176940.538820:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176940.538824:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176940.538827:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176940.538830:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176940.538832:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176940.538842:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176940.538846:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538854:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176940.538858:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538865:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.538866:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176940.538870:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176940.538874:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176940.538877:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176940.538879:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301836299
00000100:00000001:1.0:1786176940.538880:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301840077
00000100:00000001:1.0:1786176940.538882:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301836299
00000100:00000001:1.0:1786176940.538883:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301840077
00000100:00000001:1.0:1786176940.538885:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176940.538886:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.538887:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176940.538890:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176940/real 1786176940]  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176940.538896:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176940.538897:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176940.538898:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176940.538900:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176940.538901:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176940.538903:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176940.538904:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176940.538910:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538914:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176940.538915:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.538916:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176940.538917:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176940.538919:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538923:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176940.538925:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538929:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176940.538931:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.538932:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176940.538933:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.538934:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176940.538935:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176940.538936:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176940.538938:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176940.538940:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176940.538943:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176940.538948:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856317056:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176940.538950:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.538952:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176940.538954:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418e00 x1872934856316800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538958:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176940.538959:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176940.538960:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176940.538962:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176940.538963:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176940.538965:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176940.538966:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176940.538968:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:1.0:1786176940.538970:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176940.538971:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176940.538972:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418e00.
00000100:00000001:1.0:1786176940.538974:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176940.538975:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.538976:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176940.538978:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419180 x1872934856316928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.538982:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176940.538983:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176940.538984:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176940.538986:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176940.538987:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176940.538988:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176940.538989:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176940.538990:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:1.0:1786176940.538991:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176940.538992:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176940.538993:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419180.
00000100:00000001:1.0:1786176940.538995:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176940.538996:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.538997:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176940.538999:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b100 x1872934856317056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.539002:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176940.539003:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176940.539004:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176940.539005:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176940.539006:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176940.539006:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176940.539007:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176940.539008:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:1.0:1786176940.539009:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176940.539010:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176940.539011:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b100.
00000100:00000001:1.0:1786176940.539012:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176940.539013:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.539014:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.539018:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176940.539019:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.539020:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176940.539022:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176940.539022:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176940.539024:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.539028:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176940.539029:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176940.539031:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176940.539032:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176940.539033:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176940.539035:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856317184:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176940.539037:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176940.539039:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176940.539040:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176940.539041:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176940.539044:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176940.539045:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176940.539047:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176940.539049:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176940.539051:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176940.539053:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856317184, portal 4
00000100:00000001:1.0:1786176940.539054:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176940.539055:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761668224 : -114547947883392 : ffff97d1ba41b480)
00000100:00000040:1.0:1786176940.539059:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.539064:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176940.539065:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176940.539067:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176940.539068:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856317184, offset 0
00000400:00000200:1.0:1786176940.539070:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176940.539073:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176940.539075:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176940.539077:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176940.539079:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176940.539081:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176940.539082:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176940.539085:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254649:254649:256:4294967295] 192.168.202.110@tcp LPNI seq info [254638:254638:8:4294967295]
00000400:00000200:1.0:1786176940.539087:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176940.539092:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176940.539095:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176940.539097:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176940.539100:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176940.539101:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786176940.539104:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176940.539106:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1786176940.539109:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176940.539110:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176940.539112:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176940.539115:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176940.539117:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176940.539118:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176940.539119:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176940.539121:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.539126:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176940.539128:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.539132:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.539133:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176940.539135:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176940.539137:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176940.539138:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176940.539140:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301836307
00000100:00000001:1.0:1786176940.539141:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301840077
00000100:00000001:1.0:1786176940.539142:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301836307
00000100:00000001:1.0:1786176940.539144:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301840077
00000100:00000001:1.0:1786176940.539146:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176940.539147:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.539148:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176940.539150:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176940/real 1786176940]  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176940.539154:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176940.539155:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176940.539156:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176940.539158:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176940.539159:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176940.539160:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176940.539162:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176940.539168:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.539172:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176940.539172:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.539173:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176940.539174:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176940.539176:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.539180:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176940.539182:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.539186:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176940.539188:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.539189:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176940.539190:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.539191:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176940.539193:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176940.539194:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176940.539196:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176940.539197:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176940.539200:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176940.539204:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856317184:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176940.539206:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.539207:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176940.539209:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b480 x1872934856317184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176940.539214:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176940.539215:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176940.539216:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176940.539217:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176940.539218:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176940.539219:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176940.539220:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176940.539221:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7800.
02000000:00000001:1.0:1786176940.539224:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176940.539225:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176940.539227:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b480.
00000100:00000001:1.0:1786176940.539229:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176940.539231:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.539233:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176940.539236:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176940.539237:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176940.539239:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176940.539252:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176941.112623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176941.112628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176941.112632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176941.112633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176941.112636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176941.112637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176941.112707:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176941.112710:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176941.112713:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176941.560755:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176941.560763:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176941.560765:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176941.560772:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176941.560777:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176941.560781:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176941.560784:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176941.560787:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176941.560790:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176942.136667:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176942.136673:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176942.136693:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176942.136694:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176942.136697:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176942.136698:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176942.136805:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176942.136808:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176942.136811:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176942.584656:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176942.584662:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176942.584664:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176942.584668:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176942.584671:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176942.584674:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176942.584677:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176942.584679:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176942.584681:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176943.160654:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176943.160662:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176943.160667:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176943.160668:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176943.160671:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176943.160674:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176943.160676:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176943.160680:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176943.160682:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176943.160684:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176943.160686:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176943.160688:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176943.160740:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176943.160744:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176943.160748:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176943.608635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176943.608643:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176943.608645:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176943.608650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176943.608654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176943.608658:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176943.608660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176943.608663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176943.608666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176944.184618:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176944.184626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176944.184641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176944.184642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176944.184647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176944.184649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176944.184703:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176944.184706:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176944.184710:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176944.376620:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301840077, at 4301840128
00000100:00000040:1.0:1786176944.376620:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301840077, at 4301840128
00000100:00000010:1.0:1786176944.376637:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1d20.
00000100:00000010:0.0:1786176944.376640:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1720.
00000100:00000001:1.0:1786176944.632629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176944.632636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176944.632639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176944.632643:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176944.632646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176944.632650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176944.632652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176944.632655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176944.632659:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176945.208639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176945.208646:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176945.208653:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176945.208654:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176945.208658:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176945.208660:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176945.208718:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176945.208721:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176945.208724:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176945.656584:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176945.656591:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176945.656593:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176945.656598:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176945.656601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176945.656604:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.656607:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176945.656609:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.656612:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176945.656632:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176945.656639:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176945.656643:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176945.656647:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176945.656651:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176945.656658:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7168
00000400:00000200:2.0:1786176945.656667:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176945.656674:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176945.656677:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176945.656680:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176945.656684:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176945.656687:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176945.656689:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176945.656693:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176945.656695:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176945.656698:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176945.656702:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176945.656704:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176945.656707:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176945.656710:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176945.656712:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176945.656714:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176945.656716:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176945.656719:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176945.656722:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176945.656725:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176945.656728:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176945.656732:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176945.656734:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176945.656737:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176945.656739:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176945.656749:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176945.656752:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176945.656757:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176945.656759:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176945.656763:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176945.656765:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176945.656769:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176945.656777:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786176945.656783:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176945.656793:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176945.656804:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176945.656809:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176945.656814:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176945.656840:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176945.656844:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176945.656847:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176945.656849:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176945.656851:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176945.656854:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176945.656856:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7168
00000400:00000200:2.0:1786176945.656860:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176945.656863:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176945.656866:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176945.656869:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176945.656871:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176945.656873:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176945.656875:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176945.656878:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176945.656879:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176945.656881:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176945.656883:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176945.656885:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176945.656889:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176945.656893:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176945.656895:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176945.656898:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176945.656900:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176945.656903:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1786176945.656904:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786176945.656907:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176945.656911:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1786176945.656911:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176945.656919:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176945.656920:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176945.656925:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176945.656933:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176945.656937:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.656940:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176945.656940:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176945.656943:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1786176945.656945:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786176945.656947:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:2.0:1786176945.656951:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.656952:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786176945.656955:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.656956:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176945.656962:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856317312:192.168.202.110@tcp:250:kworker.0
00000100:00000010:2.0:1786176945.656963:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786176945.656968:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786176945.656970:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176945.656975:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786176945.656975:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176945.656979:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.656980:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.656986:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786176945.656986:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176945.656991:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:1.0:1786176945.656994:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:2.0:1786176945.656996:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.656998:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176945.657005:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:2.0:1786176945.657005:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786176945.657012:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176945.657016:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000100:00100000:2.0:1786176945.657021:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786176945.657023:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176945.657031:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856317312, portal 25
00000100:00000001:1.0:1786176945.657035:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786176945.657035:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176945.657039:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000001:2.0:1786176945.657041:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176945.657048:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176945.657051:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.657070:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176945.657077:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176945.657080:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176945.657084:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856317312, offset 0
00000400:00000200:1.0:1786176945.657094:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176945.657096:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176945.657100:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786176945.657103:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786176945.657107:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176945.657110:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176945.657112:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786176945.657113:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1786176945.657120:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786176945.657123:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1786176945.657127:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7168
00000400:00000200:1.0:1786176945.657130:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176945.657136:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786176945.657138:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176945.657140:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786176945.657146:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1786176945.657147:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254650:254650:256:4294967295] 192.168.202.110@tcp LPNI seq info [254639:254639:8:4294967295]
00000100:00080000:2.0:1786176945.657153:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786176945.657156:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786176945.657162:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176945.657167:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176945.657171:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1786176945.657174:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176945.657178:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:1.0:1786176945.657179:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000001:2.0:1786176945.657182:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000010:1.0:1786176945.657186:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000020:00000040:2.0:1786176945.657186:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000800:00000200:1.0:1786176945.657208:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786176945.657209:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:1.0:1786176945.657212:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000020:00000040:2.0:1786176945.657214:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176945.657220:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:1.1:1786176945.657220:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786176945.657226:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:1.1:1786176945.657227:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000100:00000040:2.0:1786176945.657235:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000100:1.0:1786176945.657236:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176945.657240:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000100:00000001:2.0:1786176945.657241:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176945.657248:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1786176945.657248:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176945.657254:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176945.657259:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:2.0:1786176945.657260:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:1.0:1786176945.657267:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000100:00080000:2.0:1786176945.657267:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176945.657271:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176945.657273:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
02000000:00000001:2.0:1786176945.657276:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176945.657281:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1786176945.657281:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176945.657286:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176945.657290:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000200:1.0:1786176945.657291:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1786176945.657292:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176945.657299:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176945.657303:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176945.657310:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000100:00000001:1.0:1786176945.657314:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000040:2.0:1786176945.657314:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176945.657319:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:1.0:1786176945.657322:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176945.657322:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176945.657327:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176945.657329:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176945.657333:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657336:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657340:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:2.0:1786176945.657340:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786176945.657346:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657348:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176945.657354:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1786176945.657354:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176945.657358:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301845197
00000100:00000010:1.0:1786176945.657366:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1d20.
00000100:00000001:2.0:1786176945.657368:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176945.657372:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657374:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:2.0:1786176945.657377:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.657381:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301845197
00000100:00000010:1.0:1786176945.657387:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1d80.
00000100:00000001:1.0:1786176945.657392:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786176945.657395:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657397:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176945.657402:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176945.657402:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176945.657407:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00080000:2.0:1786176945.657411:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00100000:1.0:1786176945.657414:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176945/real 1786176945]  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176945.657415:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176945.657419:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176945.657427:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176945.657433:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7168
00000400:00000001:1.0:1786176945.657443:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:2.0:1786176945.657443:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000001:1.0:1786176945.657446:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176945.657449:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:2.0:1786176945.657450:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1786176945.657452:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000100:00080000:2.0:1786176945.657455:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:1.0:1786176945.657456:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176945.657459:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:2.0:1786176945.657463:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:1.0:1786176945.657464:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176945.657469:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176945.657475:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:1.0:1786176945.657479:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176945.657481:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176945.657488:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176945.657493:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000040:2.0:1786176945.657493:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:1.0:1786176945.657497:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1786176945.657498:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176945.657500:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176945.657502:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786176945.657503:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00100000:1.0:1786176945.657508:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.657533:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176945.657537:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.657542:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176945.657545:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1786176945.657545:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176945.657547:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176945.657549:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1786176945.657552:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176945.657553:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176945.657556:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176945.657560:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:2.0:1786176945.657563:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00080000:1.0:1786176945.657566:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786176945.657568:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176945.657570:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786176945.657575:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176945.657577:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176945.657581:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176945.657588:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00100000:1.0:1786176945.657593:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856317312:192.168.202.110@tcp:250:kworker.0
00000100:00080000:2.0:1786176945.657596:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176945.657598:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176945.657601:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657602:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1786176945.657607:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1786176945.657608:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856317312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176945.657613:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176945.657615:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176945.657619:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000001:2.0:1786176945.657620:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176945.657622:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
00000100:00000001:2.0:1786176945.657624:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786176945.657626:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176945.657629:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000008:00100000:2.0:1786176945.657629:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000020:00000040:1.0:1786176945.657631:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176945.657637:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000008:00000001:2.0:1786176945.657637:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176945.657641:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786176945.657642:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.657647:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000010:2.0:1786176945.657648:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
02000000:00000001:1.0:1786176945.657650:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176945.657654:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176945.657654:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176945.657659:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176945.657662:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786176945.657662:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176945.657665:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657666:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176945.657672:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176945.657675:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176945.657676:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:2.0:1786176945.657678:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657679:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657685:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176945.657686:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176945.657687:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000010:2.0:1786176945.657688:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000040:1.0:1786176945.657690:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176945.657694:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657702:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176945.657704:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.657707:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00100000:2.0:1786176945.657707:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786176945.657711:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176945.657715:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176945.657722:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856317440:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786176945.657728:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176945.657729:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176945.657733:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1786176945.657734:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.657735:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.657741:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176945.657744:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176945.657746:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176945.657746:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176945.657754:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176945.657758:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176945.657762:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176945.657767:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856317440, portal 10
00000100:00000001:2.0:1786176945.657768:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657769:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176945.657772:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00080000:2.0:1786176945.657775:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1786176945.657781:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176945.657782:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176945.657787:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176945.657791:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176945.657793:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786176945.657796:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:1.0:1786176945.657797:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176945.657800:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00080000:2.0:1786176945.657801:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7168
00000100:00000200:1.0:1786176945.657803:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856317440, offset 0
00000400:00000200:2.0:1786176945.657812:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176945.657813:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786176945.657821:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1786176945.657822:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786176945.657827:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786176945.657828:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176945.657834:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1786176945.657835:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176945.657838:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1786176945.657841:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176945.657843:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:2.0:1786176945.657847:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176945.657848:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1786176945.657852:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176945.657854:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254651:254651:256:4294967295] 192.168.202.110@tcp LPNI seq info [254640:254640:8:4294967295]
00000020:00000001:2.0:1786176945.657857:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:1.0:1786176945.657861:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786176945.657861:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176945.657865:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786176945.657870:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:2.0:1786176945.657871:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176945.657877:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000800:00000200:1.0:1786176945.657878:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176945.657883:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000020:00000001:2.0:1786176945.657884:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000800:00000200:1.0:1786176945.657889:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1786176945.657892:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:1.0:1786176945.657894:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000100:00000001:2.0:1786176945.657896:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1786176945.657901:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1786176945.657903:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:1.1:1786176945.657905:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000100:00000001:2.0:1786176945.657909:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000100:1.0:1786176945.657912:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176945.657916:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000020:00000040:2.0:1786176945.657916:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000400:00000200:1.0:1786176945.657922:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:2.0:1786176945.657926:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1786176945.657929:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1786176945.657933:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176945.657935:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176945.657938:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
02000000:00000001:2.0:1786176945.657938:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176945.657943:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1786176945.657944:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1786176945.657945:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176945.657952:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176945.657958:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176945.657959:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176945.657964:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1786176945.657964:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176945.657972:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657974:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657977:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786176945.657978:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.657980:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176945.657986:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000100:00000010:2.0:1786176945.657986:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000400:00000200:1.0:1786176945.657992:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786176945.657993:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176945.657994:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301841418
00000100:00000001:1.0:1786176945.658001:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301845197
02000000:00000001:2.0:1786176945.658001:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176945.658006:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301841418
02000000:00000001:2.0:1786176945.658006:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.658010:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301845197
00000100:00000001:2.0:1786176945.658012:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786176945.658014:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176945.658016:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176945.658017:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.658021:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000001:2.0:1786176945.658022:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176945.658024:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176945/real 1786176945]  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1786176945.658030:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000400:00000001:1.0:1786176945.658032:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176945.658035:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:2.0:1786176945.658037:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176945.658038:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176945.658042:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176945.658044:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176945.658046:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00100000:2.0:1786176945.658049:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1786176945.658051:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176945.658083:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176945.658083:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176945.658090:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.658093:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176945.658095:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658097:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176945.658099:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786176945.658101:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176945.658105:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658132:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176945.658135:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176945.658137:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176945.658142:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7179)
00000100:00000001:1.0:1786176945.658143:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176945.658149:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658150:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176945.658151:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658152:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176945.658154:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176945.658156:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176945.658158:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176945.658160:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176945.658164:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176945.658170:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856317440:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176945.658174:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176945.658176:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658180:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176945.658181:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.658182:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176945.658184:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176945.658185:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176945.658187:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856317568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176945.658189:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176945.658190:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176945.658191:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.658206:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176945.658210:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176945.658211:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176945.658214:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176945.658215:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176945.658218:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176945.658220:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856317568, portal 10
00000100:00000001:1.0:1786176945.658221:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176945.658222:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:1.0:1786176945.658226:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658231:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176945.658232:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176945.658234:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176945.658235:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856317568, offset 0
00000400:00000200:1.0:1786176945.658237:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176945.658240:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176945.658242:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176945.658244:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176945.658246:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176945.658248:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176945.658250:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176945.658252:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254652:254652:256:4294967295] 192.168.202.110@tcp LPNI seq info [254641:254641:8:4294967295]
00000400:00000200:1.0:1786176945.658255:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176945.658259:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176945.658262:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176945.658264:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176945.658268:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176945.658271:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786176945.658276:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176945.658278:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:1.0:1786176945.658282:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176945.658284:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176945.658287:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176945.658290:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176945.658294:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176945.658296:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176945.658299:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176945.658303:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658311:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176945.658315:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658321:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.658322:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176945.658325:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176945.658327:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176945.658330:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176945.658331:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301841543
00000100:00000001:1.0:1786176945.658333:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301845197
00000100:00000001:1.0:1786176945.658334:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301841543
00000100:00000001:1.0:1786176945.658336:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301845197
00000100:00000001:1.0:1786176945.658338:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176945.658339:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.658340:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176945.658343:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176945/real 1786176945]  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176945.658348:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176945.658349:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176945.658350:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176945.658352:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176945.658353:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176945.658354:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176945.658356:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176945.658362:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658366:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176945.658367:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658368:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176945.658369:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176945.658371:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658375:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176945.658378:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658382:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176945.658383:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658384:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176945.658385:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658386:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176945.658388:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176945.658390:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176945.658392:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176945.658393:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176945.658396:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176945.658401:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856317568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176945.658403:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658404:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176945.658406:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856317440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658410:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176945.658411:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176945.658413:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176945.658414:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176945.658415:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176945.658416:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176945.658417:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176945.658418:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:1.0:1786176945.658420:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176945.658420:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176945.658421:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:1.0:1786176945.658423:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176945.658424:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658425:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176945.658427:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856317568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658431:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176945.658432:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176945.658433:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176945.658434:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176945.658435:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176945.658436:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176945.658437:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176945.658438:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:1.0:1786176945.658440:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176945.658440:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176945.658441:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:1.0:1786176945.658442:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176945.658443:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658444:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658446:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176945.658447:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.658448:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176945.658449:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176945.658450:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176945.658452:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658456:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176945.658457:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.658459:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176945.658460:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176945.658461:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176945.658463:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856317696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176945.658465:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176945.658466:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176945.658467:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.658468:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176945.658470:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176945.658472:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176945.658474:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176945.658476:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176945.658478:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176945.658480:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856317696, portal 4
00000100:00000001:1.0:1786176945.658481:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176945.658482:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000040:1.0:1786176945.658486:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658491:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176945.658492:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176945.658494:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176945.658495:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856317696, offset 0
00000400:00000200:1.0:1786176945.658498:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176945.658500:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176945.658502:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176945.658504:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176945.658506:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176945.658507:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176945.658524:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176945.658526:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254653:254653:256:4294967295] 192.168.202.110@tcp LPNI seq info [254642:254642:8:4294967295]
00000400:00000200:1.0:1786176945.658529:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176945.658534:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176945.658537:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176945.658540:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176945.658542:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176945.658544:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786176945.658547:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176945.658548:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:1.0:1786176945.658551:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176945.658552:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176945.658554:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176945.658557:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176945.658559:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176945.658560:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176945.658562:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176945.658564:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658569:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176945.658572:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658576:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.658577:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176945.658579:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176945.658581:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176945.658583:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176945.658585:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301841419
00000100:00000001:1.0:1786176945.658586:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301845197
00000100:00000001:1.0:1786176945.658588:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301841419
00000100:00000001:1.0:1786176945.658589:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301845197
00000100:00000001:1.0:1786176945.658590:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176945.658591:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.658593:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176945.658595:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176945/real 1786176945]  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176945.658599:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176945.658600:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176945.658602:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176945.658603:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176945.658604:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176945.658605:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176945.658607:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176945.658613:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658617:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176945.658618:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658619:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176945.658620:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176945.658622:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658627:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176945.658629:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658633:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176945.658634:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658635:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176945.658636:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658637:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176945.658638:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176945.658640:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176945.658641:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176945.658643:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176945.658645:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176945.658650:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856317696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176945.658652:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176945.658654:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658658:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176945.658659:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.658660:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176945.658661:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176945.658663:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176945.658665:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856317824:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176945.658667:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176945.658668:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176945.658669:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176945.658671:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176945.658672:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176945.658674:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176945.658675:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176945.658677:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176945.658679:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176945.658681:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856317824, portal 4
00000100:00000001:1.0:1786176945.658682:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176945.658683:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:1.0:1786176945.658687:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658691:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176945.658693:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176945.658694:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176945.658700:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856317824, offset 0
00000400:00000200:1.0:1786176945.658703:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176945.658705:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176945.658707:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176945.658709:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176945.658711:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176945.658713:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176945.658714:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176945.658716:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254654:254654:256:4294967295] 192.168.202.110@tcp LPNI seq info [254643:254643:8:4294967295]
00000400:00000200:1.0:1786176945.658719:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176945.658723:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176945.658726:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176945.658728:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176945.658730:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176945.658732:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1786176945.658735:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176945.658737:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:1.0:1786176945.658739:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176945.658741:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176945.658743:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176945.658745:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176945.658747:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176945.658748:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176945.658749:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176945.658752:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658757:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176945.658759:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658763:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.658764:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176945.658766:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176945.658768:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176945.658769:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176945.658771:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301841427
00000100:00000001:1.0:1786176945.658772:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301845197
00000100:00000001:1.0:1786176945.658774:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301841427
00000100:00000001:1.0:1786176945.658775:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301845197
00000100:00000001:1.0:1786176945.658777:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176945.658778:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.658779:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176945.658781:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176945/real 1786176945]  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176945.658785:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176945.658786:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176945.658787:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176945.658788:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176945.658789:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176945.658791:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176945.658793:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176945.658798:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658802:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176945.658803:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658804:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176945.658805:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176945.658807:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658811:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176945.658813:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658817:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176945.658818:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658819:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176945.658820:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658821:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176945.658822:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176945.658823:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176945.658825:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176945.658827:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176945.658829:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176945.658834:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856317824:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176945.658836:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658837:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176945.658840:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856317696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658844:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176945.658845:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176945.658846:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176945.658848:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176945.658848:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176945.658849:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176945.658850:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176945.658851:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:1.0:1786176945.658853:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176945.658854:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176945.658855:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:1.0:1786176945.658857:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176945.658858:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658859:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176945.658861:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856317824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176945.658865:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176945.658866:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176945.658867:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176945.658868:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176945.658868:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176945.658869:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176945.658870:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176945.658871:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:1.0:1786176945.658872:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176945.658873:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176945.658874:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:1.0:1786176945.658875:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176945.658876:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658877:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176945.658879:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176945.658879:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176945.658881:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176945.658882:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176946.232622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176946.232628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176946.232632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176946.232633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176946.232636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176946.232637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176946.232885:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176946.232890:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176946.232893:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176946.680593:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176946.680599:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176946.680601:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176946.680606:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176946.680609:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176946.680612:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176946.680614:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176946.680616:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176946.680618:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176947.256619:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176947.256626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176947.256633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176947.256635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176947.256639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176947.256640:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176947.256693:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176947.256697:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176947.256706:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176947.704585:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176947.704591:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176947.704593:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176947.704600:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176947.704604:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176947.704608:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176947.704611:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176947.704614:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176947.704617:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176948.280636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176948.280641:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176948.280646:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176948.280647:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176948.280649:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176948.280650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176948.280709:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176948.280713:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176948.280717:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176948.728594:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176948.728600:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176948.728602:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176948.728606:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176948.728609:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176948.728613:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176948.728615:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176948.728617:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176948.728619:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176949.304585:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176949.304590:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176949.304594:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176949.304595:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176949.304597:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176949.304598:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176949.304640:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176949.304643:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176949.304646:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786176949.496593:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301845197, at 4301845248
00000100:00000040:0.0:1786176949.496602:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301845197, at 4301845248
00000100:00000010:1.0:1786176949.496612:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1d80.
00000100:00000010:0.0:1786176949.496615:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1d20.
00000100:00000001:1.0:1786176949.752612:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176949.752622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176949.752625:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176949.752631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176949.752634:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176949.752637:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176949.752640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176949.752643:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176949.752645:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176950.328656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176950.328663:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176950.328665:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176950.328668:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176950.328672:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176950.328674:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176950.328676:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176950.328678:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176950.328680:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176950.328681:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176950.328682:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176950.328683:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176950.328685:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176950.328686:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176950.328688:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176950.328689:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176950.328692:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176950.328693:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176950.328801:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176950.328807:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176950.328810:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176950.776588:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176950.776594:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176950.776596:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176950.776601:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176950.776605:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176950.776608:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786176950.776611:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176950.776611:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176950.776615:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786176950.776618:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1786176950.776619:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.776622:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176950.776626:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176950.776629:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176950.776636:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7174
00000400:00000200:0.0:1786176950.776643:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176950.776648:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176950.776651:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176950.776653:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.776655:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176950.776658:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.776659:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176950.776663:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176950.776664:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786176950.776667:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176950.776669:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786176950.776671:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786176950.776672:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786176950.776674:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.776675:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176950.776677:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.776678:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176950.776680:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786176950.776682:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176950.776683:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176950.776685:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176950.776688:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176950.776690:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.776691:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786176950.776692:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176950.776700:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41ad80.
00000020:00000040:0.0:1786176950.776702:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786176950.776705:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176950.776706:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.776709:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176950.776711:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176950.776715:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176950.776721:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6000.
00000100:00000001:0.0:1786176950.776727:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176950.776734:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176950.776741:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176950.776745:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176950.776749:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034e2c676] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176950.776769:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176950.776771:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176950.776773:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786176950.776774:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176950.776776:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176950.776778:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176950.776780:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7174
00000400:00000200:0.0:1786176950.776783:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176950.776785:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176950.776787:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176950.776789:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.776791:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176950.776792:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1786176950.776793:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.776797:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176950.776799:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176950.776799:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1786176950.776803:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:1.0:1786176950.776804:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1786176950.776805:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176950.776808:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786176950.776810:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786176950.776813:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786176950.776817:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176950.776817:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176950.776821:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176950.776825:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.776828:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176950.776833:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786176950.776839:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176950.776844:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176950.776845:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.776846:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176950.776848:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786176950.776851:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786176950.776854:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176950.776854:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176950.776860:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176950.776861:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.776864:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00100000:1.0:1786176950.776868:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856317952:192.168.202.110@tcp:250:kworker.0
00000008:00100000:0.0:1786176950.776869:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1786176950.776876:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:0.0:1786176950.776877:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.776880:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176950.776881:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:0.0:1786176950.776883:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41aa00.
02000000:00000001:1.0:1786176950.776885:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786176950.776887:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786176950.776889:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1786176950.776891:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176950.776893:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.776894:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176950.776895:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:1.0:1786176950.776897:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:0.0:1786176950.776898:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176950.776904:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176950.776905:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000100:00000001:0.0:1786176950.776910:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176950.776915:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:0.0:1786176950.776916:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786176950.776924:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000100:00000001:0.0:1786176950.776925:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176950.776929:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176950.776934:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da0255a4] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1786176950.776936:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000001:0.0:1786176950.776944:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176950.776946:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:1.0:1786176950.776947:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856317952, portal 25
00000100:00080000:0.0:1786176950.776949:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1786176950.776951:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176950.776952:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786176950.776954:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761666432 : -114547947885184 : ffff97d1ba41ad80)
00000100:00080000:0.0:1786176950.776955:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176950.776960:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176950.776965:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7174
00000100:00000040:1.0:1786176950.776965:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786176950.776974:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176950.776980:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176950.776984:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176950.776985:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786176950.776988:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176950.776991:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786176950.776992:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:1.0:1786176950.776994:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000040:0.0:1786176950.776997:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:1.0:1786176950.776999:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856317952, offset 0
00000100:00000001:0.0:1786176950.777000:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176950.777006:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176950.777007:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786176950.777008:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176950.777011:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:1.0:1786176950.777011:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:0.0:1786176950.777015:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786176950.777018:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786176950.777020:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.777023:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176950.777024:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786176950.777027:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.777030:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176950.777031:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:1.0:1786176950.777033:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:0.0:1786176950.777037:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176950.777040:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176950.777042:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1786176950.777043:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176950.777047:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786176950.777049:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:0.0:1786176950.777050:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.777053:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:1.0:1786176950.777054:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000008:00100000:0.0:1786176950.777056:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1786176950.777059:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000008:00000001:0.0:1786176950.777060:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.777064:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176950.777069:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419500.
00000400:00000200:1.0:1786176950.777069:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254655:254655:256:4294967295] 192.168.202.110@tcp LPNI seq info [254644:254644:8:4294967295]
00000020:00000040:0.0:1786176950.777073:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786176950.777077:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176950.777080:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.777084:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1786176950.777085:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786176950.777086:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176950.777089:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176950.777092:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000100:00000001:0.0:1786176950.777095:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176950.777102:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176950.777109:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786176950.777115:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176950.777118:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176950.777122:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:0.0:1786176950.777124:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b6718a9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:1.0:1786176950.777128:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000100:00000001:0.0:1786176950.777132:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176950.777135:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.0:1786176950.777140:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1786176950.777141:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000200:1.0:1786176950.777157:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:0.0:1786176950.777158:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176950.777163:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:1.1:1786176950.777167:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786176950.777168:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.1:1786176950.777173:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000100:00080000:0.0:1786176950.777174:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7174
00000400:00000200:0.0:1786176950.777184:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000100:1.0:1786176950.777184:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176950.777188:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:0.0:1786176950.777189:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176950.777193:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176950.777195:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176950.777196:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1786176950.777198:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176950.777203:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176950.777205:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1786176950.777206:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176950.777209:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:1.0:1786176950.777211:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000020:00000040:0.0:1786176950.777212:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786176950.777217:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000010:1.0:1786176950.777217:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000020:00000040:0.0:1786176950.777219:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:1.0:1786176950.777222:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:0.0:1786176950.777223:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786176950.777226:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000200:1.0:1786176950.777230:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176950.777233:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.777238:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176950.777243:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.777246:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176950.777248:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1786176950.777249:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786176950.777256:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:1.0:1786176950.777257:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176950.777258:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176950.777263:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176950.777267:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176950.777271:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.777271:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.777277:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786176950.777278:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176950.777282:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786176950.777284:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.777285:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.777286:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176950.777290:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a680.
00000400:00000200:1.0:1786176950.777291:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1786176950.777295:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176950.777296:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301850317
02000000:00000001:0.0:1786176950.777301:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176950.777303:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.777305:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:1.0:1786176950.777305:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1d80.
02000000:00000001:0.0:1786176950.777308:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176950.777310:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.777313:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000010:0.0:1786176950.777316:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:1.0:1786176950.777320:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301850317
00000100:00000001:0.0:1786176950.777321:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786176950.777327:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1b40.
00000100:00100000:0.0:1786176950.777329:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176950.777331:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1786176950.777337:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176950.777343:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.777344:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176950.777346:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1786176950.777348:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176950.777353:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176950/real 1786176950]  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176950.777354:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000911ee1e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176950.777383:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176950.777384:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00080000:0.0:1786176950.777386:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:1.0:1786176950.777387:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176950.777391:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000100:00080000:0.0:1786176950.777392:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1786176950.777394:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000100:00000001:0.0:1786176950.777395:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:1.0:1786176950.777397:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00080000:0.0:1786176950.777399:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176950.777401:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1786176950.777403:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176950.777408:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7174
00000100:00000200:1.0:1786176950.777409:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786176950.777414:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176950.777420:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176950.777425:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:1.0:1786176950.777430:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176950.777431:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.777436:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176950.777440:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.777445:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176950.777450:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176950.777452:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:1.0:1786176950.777452:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176950.777456:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1786176950.777457:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176950.777460:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:1.0:1786176950.777461:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000001:0.0:1786176950.777464:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176950.777465:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1786176950.777470:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00100000:1.0:1786176950.777474:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176950.777476:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176950.777482:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176950.777486:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176950.777488:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176950.777489:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176950.777492:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000040:1.0:1786176950.777494:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786176950.777497:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176950.777503:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176950.777507:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176950.777529:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176950.777532:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.777533:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176950.777534:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.777536:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176950.777537:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:0.0:1786176950.777540:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1786176950.777540:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176950.777545:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
02000000:00000001:0.0:1786176950.777546:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.777549:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1786176950.777550:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176950.777554:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000040:1.0:1786176950.777557:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1786176950.777558:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176950.777562:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176950.777569:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419c00.
00000100:00100000:1.0:1786176950.777571:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856317952:192.168.202.110@tcp:250:kworker.0
00000020:00000040:0.0:1786176950.777573:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786176950.777579:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176950.777580:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.777580:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176950.777586:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786176950.777586:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1786176950.777590:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176950.777593:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176950.777593:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41ad80 x1872934856317952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786176950.777600:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000100:00000001:0.0:1786176950.777605:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.777606:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176950.777609:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176950.777612:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
00000100:00100000:0.0:1786176950.777614:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786176950.777616:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176950.777619:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176950.777622:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:0.0:1786176950.777623:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000020:00000001:1.0:1786176950.777625:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00000001:0.0:1786176950.777627:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176950.777629:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6000.
00000100:00000040:0.0:1786176950.777633:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cf890c1c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786176950.777634:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176950.777638:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176950.777640:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41ad80.
00000100:00000001:1.0:1786176950.777646:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786176950.777650:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.777650:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.777655:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176950.777657:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7184)
00000100:00000001:1.0:1786176950.777662:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176950.777664:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.777666:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176950.777667:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176950.777668:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176950.777672:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.777678:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176950.777679:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176950.777681:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176950.777682:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176950.777684:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176950.777686:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176950.777688:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176950.777689:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176950.777691:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176950.777692:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176950.777694:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176950.777695:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176950.777698:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176950.777701:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176950.777703:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176950.777705:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856318080, portal 10
00000100:00000001:1.0:1786176950.777707:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176950.777708:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761665536 : -114547947886080 : ffff97d1ba41aa00)
00000100:00000040:1.0:1786176950.777713:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.777717:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176950.777719:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176950.777720:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176950.777722:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856318080, offset 0
00000400:00000200:1.0:1786176950.777725:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176950.777727:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176950.777730:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176950.777732:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176950.777734:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176950.777736:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176950.777738:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176950.777740:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254656:254656:256:4294967295] 192.168.202.110@tcp LPNI seq info [254645:254645:8:4294967295]
00000400:00000200:1.0:1786176950.777742:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176950.777747:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176950.777750:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176950.777752:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176950.777755:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176950.777757:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786176950.777760:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176950.777762:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1786176950.777765:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176950.777766:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176950.777768:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176950.777770:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176950.777773:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176950.777774:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176950.777776:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176950.777778:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.777784:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176950.777787:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.777795:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.777797:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176950.777800:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176950.777805:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176950.777809:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176950.777811:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301846538
00000100:00000001:1.0:1786176950.777814:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301850317
00000100:00000001:1.0:1786176950.777816:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301846538
00000100:00000001:1.0:1786176950.777818:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301850317
00000100:00000001:1.0:1786176950.777820:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176950.777823:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.777825:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176950.777829:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176950/real 1786176950]  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176950.777838:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176950.777840:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176950.777842:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176950.777844:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176950.777845:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176950.777846:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176950.777849:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176950.777854:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.777859:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176950.777860:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.777861:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176950.777862:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176950.777864:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.777868:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176950.777871:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.777876:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176950.777877:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.777878:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176950.777879:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.777881:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176950.777882:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176950.777884:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176950.777885:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176950.777887:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176950.777889:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176950.777896:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176950.777900:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176950.777904:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.777911:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176950.777913:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176950.777915:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176950.777917:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176950.777919:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176950.777922:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318208:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176950.777926:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176950.777928:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176950.777929:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176950.777931:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176950.777935:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176950.777937:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176950.777941:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176950.777944:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176950.777946:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176950.777948:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856318208, portal 10
00000100:00000001:1.0:1786176950.777950:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176950.777951:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761660160 : -114547947891456 : ffff97d1ba419500)
00000100:00000040:1.0:1786176950.777955:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.777960:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176950.777962:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176950.777963:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176950.777964:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856318208, offset 0
00000400:00000200:1.0:1786176950.777967:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176950.777970:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176950.777972:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176950.777974:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176950.777976:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176950.777978:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176950.777979:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176950.777981:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254657:254657:256:4294967295] 192.168.202.110@tcp LPNI seq info [254646:254646:8:4294967295]
00000400:00000200:1.0:1786176950.777984:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176950.777988:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176950.777991:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176950.777993:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176950.777996:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176950.777998:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786176950.778000:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176950.778002:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1786176950.778005:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176950.778006:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176950.778008:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176950.778010:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176950.778013:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176950.778014:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176950.778016:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176950.778018:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778024:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176950.778026:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778030:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.778031:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176950.778033:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176950.778035:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176950.778037:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176950.778038:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301846663
00000100:00000001:1.0:1786176950.778040:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301850317
00000100:00000001:1.0:1786176950.778042:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301846663
00000100:00000001:1.0:1786176950.778043:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301850317
00000100:00000001:1.0:1786176950.778045:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176950.778046:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.778047:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176950.778049:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176950/real 1786176950]  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176950.778054:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176950.778055:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176950.778056:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176950.778058:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176950.778059:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176950.778060:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176950.778062:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176950.778071:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778078:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176950.778079:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778081:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176950.778083:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176950.778087:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778094:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176950.778099:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778106:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176950.778109:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778111:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176950.778112:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778114:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176950.778116:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176950.778118:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176950.778120:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176950.778122:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176950.778125:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176950.778130:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318208:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176950.778133:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176950.778135:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778140:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176950.778141:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176950.778156:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176950.778157:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176950.778159:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176950.778161:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318336:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176950.778163:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176950.778165:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176950.778166:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176950.778167:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176950.778170:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176950.778171:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176950.778174:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176950.778175:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176950.778178:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176950.778180:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856318336, portal 4
00000100:00000001:1.0:1786176950.778181:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176950.778182:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761664640 : -114547947886976 : ffff97d1ba41a680)
00000100:00000040:1.0:1786176950.778187:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778191:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176950.778193:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176950.778194:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176950.778196:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856318336, offset 0
00000400:00000200:1.0:1786176950.778198:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176950.778201:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176950.778203:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176950.778205:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176950.778207:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176950.778209:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176950.778210:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176950.778213:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254658:254658:256:4294967295] 192.168.202.110@tcp LPNI seq info [254647:254647:8:4294967295]
00000400:00000200:1.0:1786176950.778215:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176950.778219:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176950.778223:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176950.778225:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176950.778227:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176950.778229:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786176950.778232:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176950.778234:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1786176950.778237:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176950.778244:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176950.778247:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176950.778249:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176950.778254:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176950.778255:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176950.778257:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176950.778259:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778264:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176950.778266:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778270:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.778272:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176950.778274:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176950.778276:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176950.778278:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176950.778279:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301846538
00000100:00000001:1.0:1786176950.778281:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301850317
00000100:00000001:1.0:1786176950.778283:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301846538
00000100:00000001:1.0:1786176950.778284:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301850317
00000100:00000001:1.0:1786176950.778285:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176950.778286:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.778288:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176950.778290:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176950/real 1786176950]  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176950.778295:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176950.778296:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176950.778298:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176950.778299:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176950.778300:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176950.778301:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176950.778303:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176950.778308:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778312:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176950.778313:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778314:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176950.778315:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176950.778317:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778321:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176950.778324:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778328:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176950.778329:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778330:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176950.778331:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778332:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176950.778333:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176950.778334:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176950.778336:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176950.778337:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176950.778340:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176950.778345:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318336:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176950.778347:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176950.778349:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778353:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176950.778354:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176950.778355:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176950.778357:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176950.778358:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176950.778359:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176950.778361:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176950.778362:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176950.778363:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176950.778364:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176950.778366:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176950.778367:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176950.778369:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176950.778371:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176950.778372:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176950.778374:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856318464, portal 4
00000100:00000001:1.0:1786176950.778376:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176950.778376:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761661952 : -114547947889664 : ffff97d1ba419c00)
00000100:00000040:1.0:1786176950.778380:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778384:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176950.778386:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176950.778387:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176950.778388:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856318464, offset 0
00000400:00000200:1.0:1786176950.778391:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176950.778393:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176950.778395:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176950.778397:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176950.778399:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176950.778400:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176950.778402:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176950.778404:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254659:254659:256:4294967295] 192.168.202.110@tcp LPNI seq info [254648:254648:8:4294967295]
00000400:00000200:1.0:1786176950.778406:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176950.778410:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176950.778413:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176950.778414:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176950.778417:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176950.778418:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1786176950.778421:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176950.778423:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1786176950.778425:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176950.778427:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176950.778428:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176950.778431:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176950.778433:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176950.778434:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176950.778435:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176950.778437:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778442:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176950.778444:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176975 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778448:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.778449:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176950.778451:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176950.778452:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176950.778454:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176950.778456:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301846546
00000100:00000001:1.0:1786176950.778458:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301850317
00000100:00000001:1.0:1786176950.778459:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301846546
00000100:00000001:1.0:1786176950.778461:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301850317
00000100:00000001:1.0:1786176950.778462:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176950.778463:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.778464:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176950.778466:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176950/real 1786176950]  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176950.778471:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176950.778471:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176950.778472:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176950.778474:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176950.778475:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176950.778476:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176950.778478:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176950.778483:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778487:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176950.778488:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778489:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176950.778489:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176950.778492:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778495:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176950.778498:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778501:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176950.778503:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778504:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176950.778505:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778506:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176950.778507:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176950.778524:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176950.778526:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176950.778528:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176950.778530:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176950.778536:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176950.778538:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778539:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176950.778541:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41aa00 x1872934856318080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778545:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176950.778547:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176950.778548:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176950.778550:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176950.778551:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176950.778552:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176950.778553:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176950.778555:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:1.0:1786176950.778557:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176950.778558:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176950.778559:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41aa00.
00000100:00000001:1.0:1786176950.778561:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176950.778561:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778563:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176950.778565:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419500 x1872934856318208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778573:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176950.778574:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176950.778575:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176950.778576:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176950.778577:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176950.778578:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176950.778579:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176950.778580:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:1.0:1786176950.778581:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176950.778581:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176950.778582:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419500.
00000100:00000001:1.0:1786176950.778584:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176950.778584:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778585:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176950.778587:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a680 x1872934856318336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778591:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176950.778592:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176950.778592:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176950.778595:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176950.778596:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176950.778596:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176950.778597:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176950.778598:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:1.0:1786176950.778599:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176950.778600:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176950.778601:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a680.
00000100:00000001:1.0:1786176950.778602:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176950.778603:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778603:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176950.778605:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419c00 x1872934856318464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176975 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176950.778609:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176950.778610:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176950.778611:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786176950.778612:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176950.778613:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176950.778614:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176950.778615:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176950.778616:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:1.0:1786176950.778617:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176950.778618:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176950.778619:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419c00.
00000100:00000001:1.0:1786176950.778620:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176950.778620:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778621:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176950.778624:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176950.778625:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176950.778626:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176950.778627:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176951.352604:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176951.352611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176951.352617:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176951.352618:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176951.352621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176951.352622:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176951.352669:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176951.352672:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176951.352675:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176951.800644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176951.800652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176951.800654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176951.800667:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176951.800671:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176951.800675:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176951.800679:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176951.800682:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176951.800685:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176952.376596:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176952.376602:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176952.376606:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176952.376607:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176952.376610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176952.376611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176952.376669:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176952.376673:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176952.376675:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.0:1786176952.696620:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:3.0:1786176952.696632:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:3.0:1786176952.696638:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1786176952.696822:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786176952.696840:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:1.0:1786176952.696841:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1786176952.696847:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786176952.696851:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176952.824606:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176952.824613:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176952.824615:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176952.824620:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176952.824624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176952.824627:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176952.824630:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176952.824633:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176952.824635:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176953.400657:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176953.400665:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176953.400670:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176953.400671:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176953.400673:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176953.400677:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176953.400679:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176953.400682:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176953.400685:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176953.400688:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176953.400689:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176953.400691:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176953.400752:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176953.400756:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176953.400759:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176953.848661:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176953.848669:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176953.848671:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176953.848676:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176953.848680:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176953.848684:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176953.848687:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176953.848690:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176953.848692:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176954.424599:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176954.424605:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176954.424609:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176954.424610:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176954.424613:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176954.424614:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176954.424676:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176954.424681:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176954.424684:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786176954.616614:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301850317, at 4301850368
00000100:00000040:0.0:1786176954.616615:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301850317, at 4301850368
00000100:00000010:1.0:1786176954.616631:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1b40.
00000100:00000010:0.0:1786176954.616638:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1d80.
00000100:00000001:1.0:1786176954.872593:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176954.872600:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176954.872602:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176954.872608:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176954.872612:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176954.872615:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176954.872618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176954.872621:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176954.872624:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176955.448681:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176955.448688:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176955.448693:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176955.448694:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176955.448697:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176955.448698:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176955.448757:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176955.448761:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176955.448764:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786176955.896685:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176955.896686:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176955.896695:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786176955.896696:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786176955.896700:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786176955.896700:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176955.896706:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:1.0:1786176955.896708:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786176955.896711:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176955.896715:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1786176955.896719:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7179
00000100:00000001:1.0:1786176955.896721:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.896726:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000400:00000200:0.0:1786176955.896730:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176955.896731:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1786176955.896738:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176955.896738:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176955.896744:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176955.896748:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176955.896751:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176955.896753:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176955.896755:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176955.896758:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176955.896760:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786176955.896762:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176955.896765:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786176955.896767:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786176955.896769:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786176955.896771:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176955.896772:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176955.896773:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176955.896775:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176955.896777:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786176955.896779:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176955.896781:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176955.896784:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176955.896787:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176955.896789:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176955.896791:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786176955.896792:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176955.896798:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41bb80.
00000020:00000040:0.0:1786176955.896801:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786176955.896806:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176955.896807:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176955.896810:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176955.896814:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176955.896819:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176955.896826:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000100:00000001:0.0:1786176955.896833:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176955.896843:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176955.896852:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176955.896857:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176955.896861:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9983a05] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176955.896887:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176955.896891:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176955.896894:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786176955.896896:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176955.896899:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176955.896902:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176955.896906:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7179
00000100:00000001:1.0:1786176955.896908:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1786176955.896912:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176955.896912:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176955.896916:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000400:00000200:0.0:1786176955.896920:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:1.0:1786176955.896924:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786176955.896926:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176955.896933:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176955.896939:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176955.896940:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176955.896941:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176955.896947:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176955.896947:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176955.896950:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176955.896952:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:1.0:1786176955.896954:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176955.896959:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00100000:1.0:1786176955.896961:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856318592:192.168.202.110@tcp:250:kworker.0
00000020:00000040:0.0:1786176955.896963:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:1.0:1786176955.896967:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:0.0:1786176955.896968:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176955.896974:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
02000000:00000001:1.0:1786176955.896974:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176955.896978:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176955.896979:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:1.0:1786176955.896982:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:0.0:1786176955.896983:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786176955.896987:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000010:1.0:1786176955.896988:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
00000100:00000001:0.0:1786176955.896990:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1786176955.896993:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176955.896994:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176955.896997:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176955.896998:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000400:00000200:1.0:1786176955.897002:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786176955.897005:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:1.0:1786176955.897008:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000100:00000001:0.0:1786176955.897009:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176955.897011:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176955.897016:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176955.897020:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176955.897022:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176955.897025:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1786176955.897025:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000008:00000001:0.0:1786176955.897030:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176955.897032:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176955.897034:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856318592, portal 25
00000100:00000010:0.0:1786176955.897035:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a300.
00000100:00000001:1.0:1786176955.897039:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000020:00000040:0.0:1786176955.897040:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786176955.897042:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761670016 : -114547947881600 : ffff97d1ba41bb80)
02000000:00000001:0.0:1786176955.897044:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176955.897047:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176955.897049:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176955.897051:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:1.0:1786176955.897053:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786176955.897054:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176955.897060:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7800.
00000100:00000001:1.0:1786176955.897064:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1786176955.897065:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176955.897068:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176955.897072:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00100000:0.0:1786176955.897074:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1786176955.897075:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856318592, offset 0
00000100:00000001:0.0:1786176955.897084:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176955.897088:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786176955.897089:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176955.897114:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b0d3569] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176955.897122:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176955.897132:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176955.897133:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:0.0:1786176955.897136:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176955.897140:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:1.0:1786176955.897141:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1786176955.897143:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1786176955.897147:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:0.0:1786176955.897148:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786176955.897153:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1786176955.897155:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786176955.897156:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:0.0:1786176955.897163:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7179
00000400:00000200:1.0:1786176955.897164:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254660:254660:256:4294967295] 192.168.202.110@tcp LPNI seq info [254649:254649:8:4294967295]
00000400:00000200:1.0:1786176955.897172:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176955.897179:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176955.897186:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786176955.897190:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176955.897196:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000010:1.0:1786176955.897196:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000100:00080000:0.0:1786176955.897202:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176955.897207:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176955.897212:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1786176955.897212:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786176955.897219:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:1.0:1786176955.897221:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:0.0:1786176955.897222:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176955.897229:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176955.897232:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786176955.897236:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:1.1:1786176955.897236:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:0.0:1786176955.897242:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000800:00000200:1.1:1786176955.897242:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:0.0:1786176955.897248:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000100:1.0:1786176955.897252:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:0.0:1786176955.897253:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000010:1.0:1786176955.897257:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000100:00000040:0.0:1786176955.897258:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176955.897262:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176955.897264:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1786176955.897270:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176955.897271:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1786176955.897275:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176955.897278:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:1.0:1786176955.897279:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176955.897282:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00080000:0.0:1786176955.897283:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176955.897287:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.897288:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:0.0:1786176955.897293:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:1.0:1786176955.897294:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786176955.897298:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176955.897304:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.897307:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786176955.897308:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:1.0:1786176955.897312:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1786176955.897313:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786176955.897322:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.897325:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176955.897327:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176955.897333:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418380.
00000100:00000001:1.0:1786176955.897333:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:0.0:1786176955.897335:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786176955.897339:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176955.897340:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176955.897342:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786176955.897343:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176955.897344:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176955.897347:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176955.897351:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1786176955.897353:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000100:00000001:1.0:1786176955.897355:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301855437
00000100:00000001:0.0:1786176955.897358:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786176955.897363:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1b40.
00000100:00100000:0.0:1786176955.897365:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176955.897372:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176955.897372:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786176955.897376:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176955.897381:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e6b901d7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.897381:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301855437
00000100:00000010:1.0:1786176955.897386:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1120.
00000100:00000001:1.0:1786176955.897389:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1786176955.897391:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176955.897396:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.897400:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1786176955.897405:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176955.897407:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176955/real 1786176955]  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786176955.897408:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176955.897415:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786176955.897418:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176955.897420:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176955.897424:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000001:1.0:1786176955.897424:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176955.897428:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:0.0:1786176955.897430:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7179
00000400:00000010:1.0:1786176955.897432:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176955.897435:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000200:0.0:1786176955.897436:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176955.897438:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000400:00000200:0.0:1786176955.897441:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176955.897443:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00080000:0.0:1786176955.897444:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:1.0:1786176955.897449:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176955.897452:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176955.897456:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176955.897460:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176955.897464:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176955.897467:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176955.897469:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000200:1.0:1786176955.897469:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786176955.897471:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176955.897474:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786176955.897479:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176955.897481:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000001:0.0:1786176955.897482:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176955.897483:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176955.897487:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176955.897488:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176955.897491:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176955.897492:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176955.897497:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00100000:1.0:1786176955.897498:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176955.897500:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176955.897505:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:1.0:1786176955.897536:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176955.897544:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.897553:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176955.897557:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.897560:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:0.0:1786176955.897562:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176955.897563:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176955.897569:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.897571:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000001:0.0:1786176955.897572:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176955.897577:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176955.897577:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:0.0:1786176955.897583:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786176955.897583:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:0.0:1786176955.897587:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176955.897590:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00080000:1.0:1786176955.897592:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000008:00000001:0.0:1786176955.897593:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176955.897597:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.897597:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000010:0.0:1786176955.897603:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b800.
00000020:00000040:0.0:1786176955.897607:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000040:1.0:1786176955.897607:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786176955.897613:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176955.897615:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176955.897620:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176955.897623:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176955.897626:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176955.897626:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856318592:192.168.202.110@tcp:250:kworker.0
02000000:00000010:0.0:1786176955.897633:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000001:1.0:1786176955.897634:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.897638:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786176955.897639:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176955.897642:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41bb80 x1872934856318592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786176955.897647:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176955.897675:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176955.897678:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.897678:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176955.897681:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000040:0.0:1786176955.897684:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000192ee9a0] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1786176955.897685:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176955.897690:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176955.897693:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000001:0.0:1786176955.897697:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1786176955.897697:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00080000:0.0:1786176955.897703:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000020:00000001:1.0:1786176955.897703:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176955.897707:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
00000100:00080000:0.0:1786176955.897708:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786176955.897710:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176955.897713:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:1.0:1786176955.897715:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000001:0.0:1786176955.897716:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176955.897721:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7179
02000000:00000001:1.0:1786176955.897721:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000400:00000200:0.0:1786176955.897727:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:1.0:1786176955.897727:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41bb80.
00000400:00000200:0.0:1786176955.897732:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176955.897735:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176955.897735:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176955.897740:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176955.897741:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176955.897746:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176955.897747:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176955.897748:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176955.897753:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176955.897755:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176955.897756:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786176955.897757:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176955.897759:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:1.0:1786176955.897759:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000020:00000001:0.0:1786176955.897760:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:0.0:1786176955.897764:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176955.897764:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176955.897769:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176955.897771:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786176955.897772:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176955.897776:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176955.897777:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786176955.897778:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176955.897781:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1786176955.897782:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786176955.897787:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176955.897790:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176955.897791:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1786176955.897792:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786176955.897793:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176955.897798:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176955.897801:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176955.897804:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786176955.897809:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.897810:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176955.897811:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.897813:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176955.897816:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418000.
02000000:00000001:1.0:1786176955.897816:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176955.897820:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1786176955.897821:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786176955.897826:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786176955.897826:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176955.897829:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176955.897832:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176955.897833:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176955.897836:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176955.897839:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.897840:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000010:0.0:1786176955.897843:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5800.
02000000:00000001:1.0:1786176955.897844:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1786176955.897846:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176955.897848:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176955.897852:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786176955.897853:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176955.897860:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
00000100:00000001:0.0:1786176955.897861:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176955.897864:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176955.897866:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176955.897868:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002e10e4f5] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176955.897875:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176955.897883:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000100:00000001:0.0:1786176955.897888:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176955.897890:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176955.897896:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856318720, portal 10
00000100:00000040:0.0:1786176955.897899:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7189)
00000100:00000001:1.0:1786176955.897899:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176955.897901:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761663744 : -114547947887872 : ffff97d1ba41a300)
00000100:00000040:1.0:1786176955.897910:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.897915:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176955.897917:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176955.897919:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176955.897921:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856318720, offset 0
00000400:00000200:1.0:1786176955.897924:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176955.897927:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176955.897931:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176955.897934:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176955.897936:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176955.897938:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176955.897940:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176955.897942:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254661:254661:256:4294967295] 192.168.202.110@tcp LPNI seq info [254650:254650:8:4294967295]
00000400:00000200:1.0:1786176955.897945:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176955.897950:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176955.897953:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176955.897956:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176955.897959:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176955.897961:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786176955.897965:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176955.897966:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1786176955.897969:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176955.897970:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176955.897972:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176955.897975:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176955.897977:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176955.897979:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176955.897981:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176955.897983:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.897989:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176955.897991:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.897996:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.897997:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176955.897999:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176955.898003:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176955.898005:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176955.898006:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301851658
00000100:00000001:1.0:1786176955.898008:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301855437
00000100:00000001:1.0:1786176955.898010:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301851658
00000100:00000001:1.0:1786176955.898012:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301855437
00000100:00000001:1.0:1786176955.898013:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176955.898015:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.898016:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176955.898019:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176955/real 1786176955]  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176955.898023:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176955.898024:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176955.898025:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176955.898026:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176955.898027:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176955.898029:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176955.898031:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176955.898036:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898045:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176955.898046:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898047:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176955.898048:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176955.898051:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898055:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176955.898057:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898061:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176955.898063:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898064:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176955.898064:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898065:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176955.898067:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176955.898069:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176955.898071:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176955.898072:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176955.898075:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176955.898081:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176955.898083:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176955.898085:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898089:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176955.898090:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176955.898092:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176955.898108:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176955.898110:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176955.898112:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176955.898115:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176955.898117:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176955.898118:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176955.898119:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176955.898122:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176955.898123:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176955.898126:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176955.898128:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176955.898130:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176955.898132:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856318848, portal 10
00000100:00000001:1.0:1786176955.898134:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176955.898135:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761655680 : -114547947895936 : ffff97d1ba418380)
00000100:00000040:1.0:1786176955.898139:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898144:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176955.898145:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176955.898147:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176955.898148:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856318848, offset 0
00000400:00000200:1.0:1786176955.898151:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176955.898153:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176955.898156:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176955.898158:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176955.898160:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176955.898161:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176955.898163:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176955.898165:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254662:254662:256:4294967295] 192.168.202.110@tcp LPNI seq info [254651:254651:8:4294967295]
00000400:00000200:1.0:1786176955.898168:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176955.898172:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176955.898176:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176955.898178:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176955.898182:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176955.898185:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786176955.898190:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176955.898193:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1786176955.898197:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176955.898199:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176955.898203:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176955.898206:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176955.898209:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176955.898211:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176955.898214:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176955.898218:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898227:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176955.898231:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898238:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.898239:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176955.898242:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176955.898244:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176955.898247:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176955.898248:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301851783
00000100:00000001:1.0:1786176955.898250:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301855437
00000100:00000001:1.0:1786176955.898252:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301851783
00000100:00000001:1.0:1786176955.898253:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301855437
00000100:00000001:1.0:1786176955.898255:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176955.898256:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.898257:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176955.898260:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176955/real 1786176955]  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176955.898266:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176955.898267:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176955.898268:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176955.898270:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176955.898271:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176955.898273:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176955.898275:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176955.898281:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898285:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176955.898286:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898287:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176955.898288:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176955.898290:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898294:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176955.898297:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898301:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176955.898302:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898303:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176955.898304:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898305:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176955.898306:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176955.898308:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176955.898310:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176955.898311:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176955.898314:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176955.898319:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176955.898322:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176955.898324:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898328:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176955.898328:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176955.898330:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176955.898331:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176955.898332:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176955.898334:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176955.898336:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176955.898337:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176955.898338:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176955.898339:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176955.898341:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176955.898343:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176955.898345:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176955.898347:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176955.898349:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176955.898351:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856318976, portal 4
00000100:00000001:1.0:1786176955.898353:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176955.898354:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761669120 : -114547947882496 : ffff97d1ba41b800)
00000100:00000040:1.0:1786176955.898358:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898363:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176955.898364:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176955.898366:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176955.898367:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856318976, offset 0
00000400:00000200:1.0:1786176955.898370:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176955.898373:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176955.898375:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176955.898378:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176955.898379:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176955.898381:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176955.898383:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176955.898385:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254663:254663:256:4294967295] 192.168.202.110@tcp LPNI seq info [254652:254652:8:4294967295]
00000400:00000200:1.0:1786176955.898388:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176955.898392:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176955.898395:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176955.898397:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176955.898400:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176955.898401:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786176955.898404:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176955.898406:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1786176955.898409:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176955.898410:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176955.898412:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176955.898415:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176955.898417:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176955.898418:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176955.898419:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176955.898422:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898427:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176955.898429:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898433:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.898434:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176955.898435:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176955.898437:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176955.898439:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176955.898441:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301851658
00000100:00000001:1.0:1786176955.898443:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301855437
00000100:00000001:1.0:1786176955.898444:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301851658
00000100:00000001:1.0:1786176955.898446:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301855437
00000100:00000001:1.0:1786176955.898447:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176955.898448:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.898449:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176955.898451:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176955/real 1786176955]  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176955.898456:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176955.898456:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176955.898457:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176955.898458:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176955.898460:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176955.898461:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176955.898463:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176955.898469:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898472:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176955.898473:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898474:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176955.898475:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176955.898477:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898481:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176955.898483:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898487:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176955.898489:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898490:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176955.898491:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898493:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176955.898494:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176955.898495:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176955.898497:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176955.898499:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176955.898501:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176955.898505:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856318976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176955.898507:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898531:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176955.898535:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a300 x1872934856318720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898542:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176955.898544:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176955.898546:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176955.898548:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176955.898550:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176955.898551:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176955.898553:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176955.898555:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7800.
02000000:00000001:1.0:1786176955.898558:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176955.898559:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176955.898561:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a300.
00000100:00000001:1.0:1786176955.898564:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176955.898565:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898567:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176955.898571:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418380 x1872934856318848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898582:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176955.898583:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176955.898584:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176955.898587:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176955.898588:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176955.898589:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176955.898590:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176955.898591:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:1.0:1786176955.898593:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176955.898594:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176955.898595:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418380.
00000100:00000001:1.0:1786176955.898597:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176955.898598:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898600:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176955.898602:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b800 x1872934856318976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898606:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176955.898606:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176955.898607:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176955.898609:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176955.898609:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176955.898611:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176955.898612:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176955.898613:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:1.0:1786176955.898614:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176955.898615:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176955.898616:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b800.
00000100:00000001:1.0:1786176955.898617:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176955.898617:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898619:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898621:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176955.898622:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.898624:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176955.898625:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176955.898626:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176955.898628:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898633:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176955.898634:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176955.898636:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176955.898637:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176955.898638:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176955.898640:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856319104:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176955.898642:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176955.898644:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176955.898645:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176955.898646:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176955.898649:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176955.898650:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176955.898653:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176955.898655:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176955.898657:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176955.898659:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856319104, portal 4
00000100:00000001:1.0:1786176955.898661:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176955.898662:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761654784 : -114547947896832 : ffff97d1ba418000)
00000100:00000040:1.0:1786176955.898666:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898671:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176955.898672:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176955.898674:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176955.898676:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856319104, offset 0
00000400:00000200:1.0:1786176955.898678:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176955.898681:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176955.898683:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176955.898685:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176955.898687:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176955.898689:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176955.898691:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176955.898693:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254664:254664:256:4294967295] 192.168.202.110@tcp LPNI seq info [254653:254653:8:4294967295]
00000400:00000200:1.0:1786176955.898696:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176955.898701:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176955.898704:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176955.898708:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176955.898712:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176955.898715:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1786176955.898719:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176955.898722:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1786176955.898726:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176955.898728:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176955.898731:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176955.898735:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176955.898738:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176955.898740:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176955.898743:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176955.898747:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898755:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176955.898759:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176980 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898766:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.898767:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176955.898769:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176955.898772:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176955.898774:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176955.898776:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301851667
00000100:00000001:1.0:1786176955.898778:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301855437
00000100:00000001:1.0:1786176955.898780:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301851667
00000100:00000001:1.0:1786176955.898781:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301855437
00000100:00000001:1.0:1786176955.898783:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176955.898785:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.898786:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176955.898789:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176955/real 1786176955]  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176955.898794:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176955.898795:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176955.898797:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176955.898799:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176955.898800:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176955.898801:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176955.898803:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176955.898809:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898812:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176955.898813:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898814:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176955.898815:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176955.898817:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898822:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176955.898824:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898830:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176955.898831:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898832:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176955.898833:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898834:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176955.898835:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176955.898837:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176955.898840:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176955.898843:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176955.898847:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176955.898855:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856319104:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176955.898860:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898862:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176955.898865:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418000 x1872934856319104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176980 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176955.898873:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176955.898875:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176955.898876:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176955.898880:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176955.898881:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176955.898883:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176955.898885:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176955.898886:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5800.
02000000:00000001:1.0:1786176955.898889:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176955.898890:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176955.898891:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418000.
00000100:00000001:1.0:1786176955.898893:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176955.898894:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898896:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176955.898898:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176955.898899:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176955.898901:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176955.898902:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176956.472617:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176956.472627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176956.472635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176956.472637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176956.472641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176956.472642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176956.472692:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176956.472698:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176956.472702:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176956.920627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176956.920636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176956.920638:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176956.920643:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176956.920647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176956.920651:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176956.920654:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176956.920657:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176956.920659:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176957.496595:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176957.496601:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176957.496606:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176957.496606:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176957.496609:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176957.496610:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176957.496666:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176957.496670:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176957.496681:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176957.944615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176957.944622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176957.944623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176957.944628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176957.944631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176957.944635:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176957.944637:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176957.944639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176957.944641:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176958.520630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176958.520641:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176958.520649:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176958.520650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176958.520654:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176958.520655:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176958.520738:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176958.520742:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176958.520745:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176958.968674:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176958.968685:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176958.968688:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176958.968695:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176958.968700:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176958.968705:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176958.968710:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176958.968714:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176958.968718:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176959.544701:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176959.544708:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176959.544714:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176959.544715:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176959.544717:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176959.544718:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176959.544805:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176959.544813:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176959.544818:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176959.736619:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301855437, at 4301855488
00000100:00000010:0.0:1786176959.736644:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1b40.
00000100:00000040:1.0:1786176959.736650:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301855437, at 4301855488
00000100:00000010:1.0:1786176959.736667:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1120.
00000100:00000001:1.0:1786176959.992680:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176959.992692:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176959.992694:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176959.992703:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176959.992707:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176959.992712:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176959.992716:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176959.992720:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176959.992724:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176960.568653:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176960.568663:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176960.568666:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176960.568669:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176960.568674:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176960.568676:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176960.568679:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176960.568681:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176960.568685:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176960.568687:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176960.568688:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176960.568690:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176960.568692:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176960.568693:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176960.568695:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176960.568697:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176960.568700:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176960.568701:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176960.568759:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176960.568762:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176960.568765:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176961.016642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1786176961.016642:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176961.016648:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176961.016650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176961.016655:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786176961.016655:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176961.016660:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786176961.016661:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176961.016666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786176961.016667:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176961.016669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176961.016673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176961.016675:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176961.016676:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176961.016686:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7184
00000400:00000200:2.0:1786176961.016696:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176961.016703:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176961.016707:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176961.016711:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.016714:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176961.016717:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.016719:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176961.016724:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176961.016726:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176961.016729:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176961.016732:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176961.016736:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176961.016739:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176961.016742:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.016745:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176961.016747:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.016749:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176961.016752:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176961.016754:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176961.016756:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176961.016759:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176961.016762:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176961.016764:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176961.016766:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176961.016768:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176961.016776:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786176961.016779:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176961.016782:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176961.016783:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176961.016786:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176961.016788:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176961.016792:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176961.016807:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786176961.016813:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176961.016821:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176961.016831:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176961.016835:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176961.016839:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176961.016856:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176961.016858:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176961.016861:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176961.016862:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176961.016863:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176961.016866:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176961.016868:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7184
00000400:00000200:2.0:1786176961.016872:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176961.016875:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176961.016877:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176961.016880:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1786176961.016880:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.016884:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176961.016885:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176961.016888:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786176961.016888:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.016893:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176961.016896:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176961.016897:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000040:1.0:1786176961.016898:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786176961.016899:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176961.016902:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176961.016906:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176961.016911:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176961.016917:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176961.016921:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176961.016921:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176961.016925:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176961.016926:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176961.016930:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1786176961.016930:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176961.016932:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
02000000:00000001:1.0:1786176961.016934:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:2.0:1786176961.016937:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:1.0:1786176961.016938:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176961.016940:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176961.016943:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:1.0:1786176961.016944:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856319232:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1786176961.016948:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176961.016949:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786176961.016953:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176961.016955:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1786176961.016955:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786176961.016959:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1786176961.016961:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786176961.016962:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:2.0:1786176961.016966:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176961.016969:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176961.016970:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
00000100:00000010:2.0:1786176961.016972:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
02000000:00000001:1.0:1786176961.016973:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786176961.016977:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176961.016981:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:1.0:1786176961.016983:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786176961.016983:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176961.016987:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176961.016988:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:1.0:1786176961.016990:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
02000000:00000001:2.0:1786176961.016990:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176961.016997:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000400:00000010:1.0:1786176961.017000:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000001:2.0:1786176961.017000:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176961.017009:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1786176961.017010:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856319232, portal 25
00000100:00000001:1.0:1786176961.017016:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176961.017018:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454181632 : -114548255369984 : ffff97d1a7edd500)
00000100:00000001:2.0:1786176961.017021:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176961.017023:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176961.017027:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176961.017030:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.017061:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176961.017065:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176961.017068:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000001:2.0:1786176961.017074:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176961.017075:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856319232, offset 0
00000100:00080000:2.0:1786176961.017077:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176961.017082:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176961.017083:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176961.017085:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176961.017088:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176961.017091:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7184
00000400:00000200:2.0:1786176961.017095:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176961.017098:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786176961.017099:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176961.017103:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176961.017105:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.017107:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176961.017108:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.017110:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176961.017113:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1786176961.017114:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176961.017116:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176961.017118:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176961.017120:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176961.017121:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786176961.017125:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:2.0:1786176961.017125:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176961.017132:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.017134:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176961.017137:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1786176961.017137:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.017143:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176961.017144:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:1.0:1786176961.017148:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:2.0:1786176961.017149:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176961.017154:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176961.017155:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786176961.017158:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:2.0:1786176961.017158:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176961.017161:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176961.017164:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.017165:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176961.017168:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1786176961.017173:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254665:254665:256:4294967295] 192.168.202.110@tcp LPNI seq info [254654:254654:8:4294967295]
00000008:00000001:2.0:1786176961.017173:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176961.017176:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176961.017181:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000400:00000200:1.0:1786176961.017182:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786176961.017186:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176961.017189:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176961.017190:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176961.017193:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176961.017196:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1786176961.017199:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1786176961.017199:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176961.017206:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786176961.017209:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176961.017210:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00100000:2.0:1786176961.017215:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:1.0:1786176961.017217:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000100:00000001:2.0:1786176961.017225:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176961.017229:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176961.017230:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176961.017237:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000040:2.0:1786176961.017238:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176961.017249:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786176961.017249:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786176961.017254:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.1:1786176961.017255:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00080000:2.0:1786176961.017260:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176961.017263:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000100:1.0:1786176961.017264:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176961.017266:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000010:1.0:1786176961.017267:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000100:00000001:2.0:1786176961.017270:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786176961.017274:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:2.0:1786176961.017274:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7184
00000400:00000200:2.0:1786176961.017279:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176961.017282:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176961.017283:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176961.017285:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.017288:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176961.017293:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:2.0:1786176961.017294:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.017298:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176961.017301:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176961.017302:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000400:00000200:1.0:1786176961.017303:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000020:00000001:2.0:1786176961.017303:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000010:1.0:1786176961.017308:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000020:00000040:2.0:1786176961.017308:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176961.017313:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176961.017315:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176961.017316:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1786176961.017319:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.017321:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1786176961.017324:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176961.017326:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.017330:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176961.017335:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176961.017342:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176961.017344:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786176961.017346:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176961.017349:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176961.017351:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176961.017356:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176961.017360:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.017363:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176961.017363:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176961.017367:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000008:00100000:2.0:1786176961.017368:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176961.017372:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.017373:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176961.017375:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176961.017380:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:2.0:1786176961.017382:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000100:00000001:1.0:1786176961.017383:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301860557
00000020:00000040:2.0:1786176961.017388:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000010:1.0:1786176961.017390:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1120.
02000000:00000001:2.0:1786176961.017392:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176961.017395:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176961.017397:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176961.017398:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176961.017400:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:2.0:1786176961.017400:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176961.017408:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786176961.017411:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.017415:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301860557
00000100:00100000:2.0:1786176961.017416:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:1.0:1786176961.017423:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1f00.
00000100:00000001:2.0:1786176961.017425:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176961.017428:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786176961.017429:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.017434:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1786176961.017436:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.017439:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.017445:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1786176961.017450:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176961.017454:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176961/real 1786176961]  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176961.017455:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176961.017464:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786176961.017467:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176961.017471:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000001:1.0:1786176961.017474:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786176961.017476:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000001:1.0:1786176961.017478:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176961.017483:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000100:00080000:2.0:1786176961.017483:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7184
00000400:00000200:1.0:1786176961.017490:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000200:2.0:1786176961.017492:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176961.017494:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176961.017500:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000400:00000200:2.0:1786176961.017500:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176961.017505:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:1.0:1786176961.017538:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176961.017544:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176961.017550:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1786176961.017551:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176961.017556:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176961.017561:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786176961.017563:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176961.017564:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.017569:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000001:2.0:1786176961.017569:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176961.017574:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786176961.017574:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176961.017580:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00100000:1.0:1786176961.017582:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176961.017585:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176961.017591:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176961.017593:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1786176961.017597:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:1.0:1786176961.017600:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176961.017605:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176961.017610:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786176961.017611:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176961.017614:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.017616:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:2.0:1786176961.017617:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176961.017618:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176961.017621:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176961.017623:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000040:2.0:1786176961.017626:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:1.0:1786176961.017627:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176961.017633:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786176961.017633:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176961.017639:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786176961.017643:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
02000000:00000001:2.0:1786176961.017644:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176961.017646:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:2.0:1786176961.017649:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1786176961.017651:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176961.017656:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176961.017662:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00100000:1.0:1786176961.017665:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856319232:192.168.202.110@tcp:250:kworker.0
00000008:00100000:2.0:1786176961.017666:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786176961.017670:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:2.0:1786176961.017672:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.017674:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786176961.017676:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176961.017678:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd500 x1872934856319232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786176961.017684:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000100:00000001:1.0:1786176961.017685:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176961.017688:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000020:00000040:2.0:1786176961.017690:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000010:1.0:1786176961.017691:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:2.0:1786176961.017694:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786176961.017696:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000001:2.0:1786176961.017697:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1786176961.017700:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000001:2.0:1786176961.017701:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000020:00000040:1.0:1786176961.017703:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176961.017705:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000020:00000001:1.0:1786176961.017706:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000001:2.0:1786176961.017709:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176961.017710:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:1.0:1786176961.017716:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176961.017719:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
02000000:00000010:2.0:1786176961.017719:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000010:1.0:1786176961.017722:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd500.
00000100:00000001:1.0:1786176961.017726:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176961.017727:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176961.017727:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.017733:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.017740:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00100000:2.0:1786176961.017740:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176961.017743:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.017748:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176961.017750:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176961.017753:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176961.017759:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176961.017772:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176961.017774:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176961.017776:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.017777:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176961.017781:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1786176961.017784:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786176961.017785:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176961.017790:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176961.017798:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176961.017803:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7194)
00000100:00100000:1.0:1786176961.017804:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856319360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176961.017811:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176961.017812:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176961.017814:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176961.017816:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176961.017818:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176961.017820:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176961.017824:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176961.017826:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176961.017828:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176961.017831:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856319360, portal 10
00000100:00000001:1.0:1786176961.017832:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176961.017833:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178048 : -114548255373568 : ffff97d1a7edc700)
00000100:00000040:1.0:1786176961.017839:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.017843:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176961.017845:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176961.017846:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176961.017848:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856319360, offset 0
00000400:00000200:1.0:1786176961.017850:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176961.017853:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176961.017856:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176961.017859:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176961.017861:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176961.017862:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176961.017864:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176961.017867:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254666:254666:256:4294967295] 192.168.202.110@tcp LPNI seq info [254655:254655:8:4294967295]
00000400:00000200:1.0:1786176961.017869:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176961.017874:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176961.017877:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176961.017879:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176961.017882:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176961.017884:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786176961.017887:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176961.017889:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1786176961.017891:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176961.017893:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176961.017895:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176961.017897:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176961.017900:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176961.017901:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176961.017903:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176961.017905:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.017910:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176961.017912:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.017916:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.017917:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176961.017919:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176961.017922:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176961.017925:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176961.017926:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301856778
00000100:00000001:1.0:1786176961.017928:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301860557
00000100:00000001:1.0:1786176961.017930:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301856778
00000100:00000001:1.0:1786176961.017931:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301860557
00000100:00000001:1.0:1786176961.017933:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176961.017934:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.017935:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176961.017937:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176961/real 1786176961]  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176961.017942:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176961.017943:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176961.017944:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176961.017945:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176961.017946:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176961.017948:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176961.017950:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176961.017955:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.017959:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176961.017960:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.017961:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176961.017962:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176961.017964:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.017968:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176961.017970:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.017975:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176961.017976:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.017978:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176961.017979:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.017980:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176961.017982:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176961.017983:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176961.017986:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176961.017987:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176961.017989:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176961.017994:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856319360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176961.017996:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176961.017998:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018002:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176961.018003:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176961.018004:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176961.018005:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176961.018007:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176961.018009:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856319488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176961.018011:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176961.018012:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176961.018013:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176961.018014:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176961.018016:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176961.018017:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176961.018019:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176961.018020:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176961.018022:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176961.018024:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856319488, portal 10
00000100:00000001:1.0:1786176961.018025:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176961.018026:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454180736 : -114548255370880 : ffff97d1a7edd180)
00000100:00000040:1.0:1786176961.018030:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018034:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176961.018036:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176961.018037:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176961.018039:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856319488, offset 0
00000400:00000200:1.0:1786176961.018041:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176961.018058:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176961.018061:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176961.018063:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176961.018065:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176961.018066:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176961.018068:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176961.018070:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254667:254667:256:4294967295] 192.168.202.110@tcp LPNI seq info [254656:254656:8:4294967295]
00000400:00000200:1.0:1786176961.018073:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176961.018077:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176961.018081:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176961.018083:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176961.018086:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176961.018088:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786176961.018090:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176961.018092:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1786176961.018094:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176961.018096:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176961.018098:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176961.018100:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176961.018102:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176961.018104:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176961.018105:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176961.018108:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018117:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176961.018119:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018124:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.018125:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176961.018127:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176961.018129:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176961.018131:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176961.018132:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301856903
00000100:00000001:1.0:1786176961.018133:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301860557
00000100:00000001:1.0:1786176961.018135:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301856903
00000100:00000001:1.0:1786176961.018137:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301860557
00000100:00000001:1.0:1786176961.018138:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176961.018139:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.018140:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176961.018143:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176961/real 1786176961]  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176961.018147:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176961.018148:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176961.018149:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176961.018151:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176961.018152:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176961.018153:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176961.018155:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176961.018160:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018164:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176961.018165:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018167:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176961.018168:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176961.018170:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018174:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176961.018176:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018180:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176961.018182:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018183:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176961.018183:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018184:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176961.018186:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176961.018187:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176961.018189:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176961.018190:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176961.018193:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176961.018197:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856319488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176961.018199:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176961.018202:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018206:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176961.018207:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176961.018208:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176961.018210:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176961.018211:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176961.018213:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856319616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176961.018215:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176961.018216:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176961.018217:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176961.018219:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176961.018221:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176961.018222:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176961.018224:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176961.018225:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176961.018227:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176961.018229:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856319616, portal 4
00000100:00000001:1.0:1786176961.018231:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176961.018232:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187904 : -114548255363712 : ffff97d1a7eded80)
00000100:00000040:1.0:1786176961.018235:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018239:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176961.018241:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176961.018242:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176961.018243:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856319616, offset 0
00000400:00000200:1.0:1786176961.018246:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176961.018248:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176961.018250:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176961.018252:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176961.018253:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176961.018255:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176961.018256:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176961.018258:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254668:254668:256:4294967295] 192.168.202.110@tcp LPNI seq info [254657:254657:8:4294967295]
00000400:00000200:1.0:1786176961.018261:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176961.018264:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176961.018267:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176961.018269:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176961.018272:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176961.018273:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786176961.018276:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176961.018278:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1786176961.018281:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176961.018282:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176961.018284:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176961.018286:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176961.018288:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176961.018289:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176961.018290:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176961.018292:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018297:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176961.018299:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018303:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.018304:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176961.018306:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176961.018308:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176961.018310:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176961.018311:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301856778
00000100:00000001:1.0:1786176961.018313:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301860557
00000100:00000001:1.0:1786176961.018314:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301856778
00000100:00000001:1.0:1786176961.018316:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301860557
00000100:00000001:1.0:1786176961.018317:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176961.018318:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.018319:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176961.018321:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176961/real 1786176961]  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176961.018325:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176961.018327:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176961.018328:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176961.018329:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176961.018330:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176961.018331:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176961.018333:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176961.018338:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018342:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176961.018343:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018344:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176961.018345:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176961.018347:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018351:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176961.018353:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018357:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176961.018358:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018359:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176961.018360:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018361:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176961.018362:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176961.018363:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176961.018365:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176961.018366:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176961.018369:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176961.018378:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856319616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176961.018380:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018382:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176961.018383:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc700 x1872934856319360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018387:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176961.018388:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176961.018389:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176961.018391:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176961.018392:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176961.018393:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176961.018394:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176961.018395:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:1.0:1786176961.018397:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176961.018397:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176961.018399:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc700.
00000100:00000001:1.0:1786176961.018400:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176961.018401:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018403:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176961.018405:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd180 x1872934856319488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018409:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176961.018409:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176961.018410:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176961.018412:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176961.018413:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176961.018414:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176961.018415:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176961.018416:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:1.0:1786176961.018417:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176961.018418:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176961.018419:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd180.
00000100:00000001:1.0:1786176961.018421:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176961.018422:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018423:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176961.018425:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eded80 x1872934856319616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018429:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176961.018430:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176961.018431:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176961.018432:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176961.018432:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176961.018433:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176961.018434:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176961.018435:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:1.0:1786176961.018436:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176961.018437:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176961.018438:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eded80.
00000100:00000001:1.0:1786176961.018439:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176961.018440:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018441:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018443:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176961.018444:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.018445:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176961.018446:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176961.018447:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176961.018449:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018453:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176961.018454:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176961.018455:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176961.018456:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176961.018457:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176961.018459:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856319744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176961.018461:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176961.018462:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176961.018463:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176961.018464:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176961.018466:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176961.018468:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176961.018471:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176961.018474:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176961.018477:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176961.018480:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856319744, portal 4
00000100:00000001:1.0:1786176961.018482:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176961.018483:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:1.0:1786176961.018490:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018497:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176961.018499:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176961.018502:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176961.018504:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856319744, offset 0
00000400:00000200:1.0:1786176961.018509:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176961.018532:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176961.018535:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176961.018537:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176961.018539:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176961.018541:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176961.018542:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176961.018545:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254669:254669:256:4294967295] 192.168.202.110@tcp LPNI seq info [254658:254658:8:4294967295]
00000400:00000200:1.0:1786176961.018547:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176961.018551:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176961.018554:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176961.018557:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176961.018560:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176961.018561:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786176961.018564:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176961.018566:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1786176961.018569:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176961.018570:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176961.018572:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176961.018575:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176961.018577:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176961.018578:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176961.018580:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176961.018583:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018588:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176961.018590:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176986 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018594:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.018596:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176961.018598:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176961.018600:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176961.018602:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176961.018604:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301856787
00000100:00000001:1.0:1786176961.018605:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301860557
00000100:00000001:1.0:1786176961.018607:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301856787
00000100:00000001:1.0:1786176961.018608:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301860557
00000100:00000001:1.0:1786176961.018610:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176961.018611:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.018612:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176961.018614:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176961/real 1786176961]  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176961.018619:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176961.018620:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176961.018622:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176961.018623:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176961.018624:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176961.018625:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176961.018628:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176961.018633:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018637:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176961.018638:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018639:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176961.018640:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176961.018642:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018646:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176961.018648:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018652:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176961.018654:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018655:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176961.018655:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018656:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176961.018658:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176961.018659:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176961.018661:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176961.018663:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176961.018665:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176961.018670:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856319744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176961.018672:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018674:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176961.018675:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856319744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176986 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176961.018680:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176961.018681:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176961.018682:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176961.018684:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176961.018685:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176961.018685:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176961.018686:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176961.018688:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0800.
02000000:00000001:1.0:1786176961.018689:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176961.018690:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176961.018691:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:1.0:1786176961.018693:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176961.018693:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018695:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176961.018697:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176961.018698:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176961.018699:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176961.018701:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176961.592626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176961.592635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176961.592642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176961.592643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176961.592647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176961.592649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176961.592698:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176961.592703:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176961.592707:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176962.040617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176962.040625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176962.040627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176962.040633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176962.040637:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176962.040641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176962.040644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176962.040648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176962.040651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176962.616612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176962.616618:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176962.616622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176962.616624:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176962.616626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176962.616629:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176962.616631:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176962.616634:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176962.616636:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176962.616638:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176962.616640:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176962.616642:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176962.616696:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176962.616700:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176962.616702:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176963.064625:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176963.064634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176963.064636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176963.064642:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176963.064646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176963.064650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176963.064654:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176963.064656:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176963.064660:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176963.640642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176963.640648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176963.640653:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176963.640655:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176963.640658:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176963.640659:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176963.640722:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176963.640726:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176963.640730:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176964.088635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176964.088644:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176964.088646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176964.088652:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176964.088656:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176964.088660:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176964.088664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176964.088666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176964.088670:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176964.664635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176964.664641:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176964.664645:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176964.664646:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176964.664650:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176964.664652:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176964.664709:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176964.664713:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176964.664716:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176964.856615:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301860557, at 4301860608
00000100:00000040:1.0:1786176964.856615:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301860557, at 4301860608
00000100:00000010:1.0:1786176964.856629:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1f00.
00000100:00000010:0.0:1786176964.856631:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1120.
00000100:00000001:1.0:1786176965.112580:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176965.112586:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176965.112588:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176965.112592:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176965.112595:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176965.112597:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176965.112600:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176965.112602:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176965.112604:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176965.688629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176965.688636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176965.688643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176965.688644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176965.688648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176965.688650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176965.688710:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176965.688714:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176965.688717:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176966.136605:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1786176966.136605:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176966.136611:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176966.136614:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176966.136615:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176966.136618:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:1.0:1786176966.136620:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786176966.136622:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176966.136626:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176966.136627:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176966.136630:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786176966.136634:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7189
00000100:00000001:1.0:1786176966.136635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176966.136639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.136642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786176966.136646:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176966.136652:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176966.136655:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176966.136657:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.136660:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176966.136663:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.136665:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176966.136668:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176966.136670:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176966.136672:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176966.136675:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176966.136677:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176966.136679:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176966.136681:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.136683:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176966.136684:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.136686:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176966.136689:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176966.136691:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176966.136693:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176966.136695:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176966.136698:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176966.136700:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176966.136701:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176966.136702:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176966.136711:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786176966.136713:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176966.136716:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176966.136717:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176966.136720:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176966.136723:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176966.136726:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176966.136732:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786176966.136739:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176966.136749:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176966.136760:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176966.136764:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176966.136769:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176966.136793:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176966.136797:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176966.136800:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176966.136801:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176966.136802:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176966.136804:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176966.136807:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7189
00000400:00000200:2.0:1786176966.136810:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176966.136812:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176966.136814:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176966.136816:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.136817:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176966.136819:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.136821:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176966.136824:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176966.136825:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176966.136827:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176966.136829:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786176966.136831:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176966.136833:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786176966.136837:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.136839:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176966.136842:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.136845:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176966.136848:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786176966.136850:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:2.0:1786176966.136852:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176966.136854:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176966.136859:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786176966.136859:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176966.136864:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1786176966.136867:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176966.136871:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176966.136878:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.136882:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176966.136883:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176966.136884:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.136889:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:2.0:1786176966.136890:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786176966.136891:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176966.136896:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1786176966.136898:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176966.136901:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856319872:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786176966.136903:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.136905:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176966.136909:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:2.0:1786176966.136910:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
02000000:00000001:1.0:1786176966.136912:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.136916:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:2.0:1786176966.136917:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000010:1.0:1786176966.136921:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:2.0:1786176966.136922:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786176966.136926:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176966.136927:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176966.136931:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176966.136933:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176966.136935:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176966.136936:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:2.0:1786176966.136942:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000400:00000010:1.0:1786176966.136943:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000100:00000001:2.0:1786176966.136949:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176966.136951:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00100000:2.0:1786176966.136957:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1786176966.136958:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856319872, portal 25
00000100:00000001:1.0:1786176966.136963:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176966.136966:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000001:2.0:1786176966.136972:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1786176966.136976:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176966.136976:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176966.136982:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.136988:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176966.137005:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176966.137010:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000001:2.0:1786176966.137011:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176966.137014:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856319872, offset 0
00000100:00080000:2.0:1786176966.137015:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176966.137020:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176966.137022:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1786176966.137025:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176966.137025:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176966.137030:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176966.137032:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7189
00000400:00000200:1.0:1786176966.137035:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176966.137039:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176966.137044:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786176966.137045:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176966.137049:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786176966.137052:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1786176966.137052:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.137056:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176966.137059:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:2.0:1786176966.137061:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.137065:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176966.137066:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:2.0:1786176966.137069:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:1.0:1786176966.137071:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000040:2.0:1786176966.137073:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786176966.137075:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786176966.137078:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254670:254670:256:4294967295] 192.168.202.110@tcp LPNI seq info [254659:254659:8:4294967295]
00000020:00000040:2.0:1786176966.137079:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786176966.137084:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786176966.137088:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1786176966.137089:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786176966.137094:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.137099:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176966.137102:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1786176966.137105:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.137109:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:1.0:1786176966.137112:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:2.0:1786176966.137112:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786176966.137117:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000010:1.0:1786176966.137118:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000100:00000001:2.0:1786176966.137120:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176966.137123:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176966.137127:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:1.0:1786176966.137128:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1786176966.137134:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176966.137135:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1786176966.137140:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000800:00000200:1.1:1786176966.137143:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000008:00100000:2.0:1786176966.137145:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000800:00000200:1.1:1786176966.137147:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000008:00000001:2.0:1786176966.137152:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1786176966.137153:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176966.137157:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000100:00000001:2.0:1786176966.137157:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176966.137165:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000400:00000200:1.0:1786176966.137166:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:2.0:1786176966.137170:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:1.0:1786176966.137177:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:2.0:1786176966.137177:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176966.137179:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176966.137183:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176966.137184:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176966.137186:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176966.137187:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
02000000:00000010:2.0:1786176966.137190:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000400:00000010:1.0:1786176966.137192:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:2.0:1786176966.137194:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.137198:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:2.0:1786176966.137202:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1786176966.137208:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176966.137211:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176966.137215:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176966.137223:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137227:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786176966.137231:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176966.137233:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176966.137235:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176966.137243:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786176966.137246:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786176966.137250:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176966.137250:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176966.137255:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786176966.137256:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176966.137260:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7189
00000100:00000001:1.0:1786176966.137262:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786176966.137264:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176966.137269:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176966.137270:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176966.137274:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301865677
00000100:00080000:2.0:1786176966.137274:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176966.137280:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000010:1.0:1786176966.137282:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1f00.
00000100:00000001:2.0:1786176966.137283:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176966.137287:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.137290:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176966.137291:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000001:2.0:1786176966.137292:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176966.137294:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786176966.137297:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176966.137299:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176966.137300:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786176966.137302:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176966.137304:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.137305:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176966.137306:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301865677
00000100:00000040:2.0:1786176966.137307:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.137311:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000010:1.0:1786176966.137312:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b11e0.
00000020:00000040:2.0:1786176966.137315:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:1.0:1786176966.137316:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00080000:2.0:1786176966.137320:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786176966.137322:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786176966.137323:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.137326:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176966.137327:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176966.137331:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000001:2.0:1786176966.137332:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:1.0:1786176966.137337:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176966/real 1786176966]  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176966.137338:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176966.137345:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000001:1.0:1786176966.137351:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000008:00100000:2.0:1786176966.137351:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000001:1.0:1786176966.137354:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176966.137358:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000008:00000001:2.0:1786176966.137358:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176966.137362:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000100:00000001:2.0:1786176966.137364:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176966.137365:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176966.137369:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176966.137372:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786176966.137372:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786176966.137378:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000200:1.0:1786176966.137384:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176966.137385:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176966.137388:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176966.137392:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176966.137394:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176966.137397:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:2.0:1786176966.137398:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.137399:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.137402:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000010:2.0:1786176966.137402:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:1.0:1786176966.137403:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176966.137409:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176966.137409:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176966.137418:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176966.137419:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176966.137427:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176966.137430:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176966.137434:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.137439:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:2.0:1786176966.137441:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137443:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.137447:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176966.137450:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.137455:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786176966.137456:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.137457:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786176966.137459:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786176966.137461:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786176966.137466:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00080000:1.0:1786176966.137469:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786176966.137470:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176966.137474:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176966.137475:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1786176966.137477:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176966.137481:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7189
00000100:00000040:1.0:1786176966.137486:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786176966.137488:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176966.137493:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176966.137497:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176966.137502:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.137506:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176966.137530:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.137533:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176966.137534:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176966.137536:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176966.137537:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00100000:1.0:1786176966.137538:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856319872:192.168.202.110@tcp:250:kworker.0
00000020:00000040:2.0:1786176966.137540:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176966.137545:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176966.137548:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1786176966.137548:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176966.137552:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176966.137555:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176966.137559:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856319872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176966.137559:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176966.137563:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176966.137567:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176966.137572:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:1.0:1786176966.137574:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00080000:2.0:1786176966.137577:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:1.0:1786176966.137578:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:2.0:1786176966.137580:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176966.137581:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:2.0:1786176966.137583:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786176966.137585:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000001:2.0:1786176966.137587:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000001:1.0:1786176966.137589:0:2391:0:(genops.c:1238:class_import_put()) Process entered
02000000:00000001:2.0:1786176966.137590:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1786176966.137592:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:2.0:1786176966.137593:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000020:00000001:1.0:1786176966.137595:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000008:00100000:2.0:1786176966.137596:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000010:1.0:1786176966.137598:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
00000008:00000001:2.0:1786176966.137600:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.137603:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000001:2.0:1786176966.137603:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.137606:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786176966.137606:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000100:00000010:1.0:1786176966.137607:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786176966.137611:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:1.0:1786176966.137612:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
02000000:00000001:2.0:1786176966.137614:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176966.137615:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786176966.137617:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.137619:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176966.137620:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176966.137623:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176966.137624:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.137626:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000010:2.0:1786176966.137628:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:1.0:1786176966.137629:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.137633:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786176966.137633:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.137635:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176966.137637:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00100000:2.0:1786176966.137641:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1786176966.137642:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137664:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176966.137667:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.137671:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176966.137674:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786176966.137675:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1786176966.137677:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176966.137679:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176966.137682:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176966.137688:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:2.0:1786176966.137688:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786176966.137691:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176966.137694:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.137698:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176966.137702:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
00000100:00000001:2.0:1786176966.137703:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.137705:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176966.137710:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786176966.137710:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7199)
00000400:00000010:1.0:1786176966.137712:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176966.137717:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176966.137719:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856320000, portal 10
00000100:00000001:1.0:1786176966.137721:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176966.137723:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:1.0:1786176966.137729:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137734:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176966.137736:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176966.137737:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176966.137738:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856320000, offset 0
00000400:00000200:1.0:1786176966.137741:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176966.137744:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176966.137746:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176966.137749:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176966.137750:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176966.137752:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176966.137754:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176966.137756:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254671:254671:256:4294967295] 192.168.202.110@tcp LPNI seq info [254660:254660:8:4294967295]
00000400:00000200:1.0:1786176966.137759:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176966.137763:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176966.137766:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176966.137769:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176966.137771:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176966.137773:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786176966.137776:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176966.137778:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786176966.137781:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176966.137782:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176966.137784:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176966.137787:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176966.137789:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176966.137791:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176966.137792:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176966.137794:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137800:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176966.137802:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137806:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.137807:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176966.137809:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176966.137812:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176966.137814:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176966.137816:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301861898
00000100:00000001:1.0:1786176966.137818:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301865677
00000100:00000001:1.0:1786176966.137819:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301861898
00000100:00000001:1.0:1786176966.137821:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301865677
00000100:00000001:1.0:1786176966.137822:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176966.137823:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.137824:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176966.137826:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176966/real 1786176966]  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176966.137832:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176966.137833:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176966.137834:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176966.137835:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176966.137836:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176966.137837:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176966.137840:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176966.137845:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137849:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176966.137850:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.137851:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176966.137852:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176966.137854:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137858:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176966.137861:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137865:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176966.137866:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.137867:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176966.137868:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.137869:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176966.137870:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176966.137872:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176966.137874:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176966.137875:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176966.137878:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176966.137882:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176966.137885:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176966.137887:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137896:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176966.137897:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.137898:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176966.137899:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176966.137901:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176966.137902:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176966.137904:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176966.137905:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176966.137906:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.137907:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176966.137909:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176966.137910:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176966.137912:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176966.137913:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176966.137915:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176966.137917:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856320128, portal 10
00000100:00000001:1.0:1786176966.137918:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176966.137919:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:1.0:1786176966.137923:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137927:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176966.137929:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176966.137930:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176966.137932:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856320128, offset 0
00000400:00000200:1.0:1786176966.137934:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176966.137936:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176966.137938:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176966.137940:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176966.137941:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176966.137942:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176966.137944:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176966.137946:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254672:254672:256:4294967295] 192.168.202.110@tcp LPNI seq info [254661:254661:8:4294967295]
00000400:00000200:1.0:1786176966.137949:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176966.137953:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176966.137956:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176966.137958:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176966.137960:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176966.137962:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786176966.137964:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176966.137966:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786176966.137968:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176966.137970:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176966.137971:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176966.137974:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176966.137975:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176966.137977:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176966.137978:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176966.137980:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137985:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176966.137987:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.137991:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.137992:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176966.138009:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176966.138012:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176966.138015:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176966.138016:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301862023
00000100:00000001:1.0:1786176966.138018:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301865677
00000100:00000001:1.0:1786176966.138019:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301862023
00000100:00000001:1.0:1786176966.138021:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301865677
00000100:00000001:1.0:1786176966.138022:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176966.138024:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.138025:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176966.138028:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176966/real 1786176966]  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176966.138034:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176966.138035:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176966.138036:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176966.138038:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176966.138039:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176966.138040:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176966.138042:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176966.138048:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138052:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176966.138053:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138054:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176966.138055:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176966.138057:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138061:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176966.138063:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138067:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176966.138069:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138070:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176966.138070:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138071:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176966.138073:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176966.138074:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176966.138077:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176966.138078:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176966.138081:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176966.138085:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176966.138088:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176966.138089:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138093:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176966.138094:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.138095:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176966.138097:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176966.138098:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176966.138100:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176966.138101:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176966.138102:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176966.138103:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.138104:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176966.138107:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176966.138108:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176966.138110:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176966.138112:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176966.138114:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176966.138116:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856320256, portal 4
00000100:00000001:1.0:1786176966.138117:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176966.138118:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:1.0:1786176966.138122:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138126:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176966.138128:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176966.138129:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176966.138131:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856320256, offset 0
00000400:00000200:1.0:1786176966.138134:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176966.138136:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176966.138139:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176966.138141:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176966.138142:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176966.138144:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176966.138146:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176966.138148:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254673:254673:256:4294967295] 192.168.202.110@tcp LPNI seq info [254662:254662:8:4294967295]
00000400:00000200:1.0:1786176966.138151:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176966.138155:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176966.138157:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176966.138159:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176966.138162:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176966.138163:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786176966.138167:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176966.138170:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786176966.138174:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176966.138176:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176966.138179:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176966.138182:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176966.138185:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176966.138187:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176966.138189:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176966.138193:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138202:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176966.138205:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138213:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.138214:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176966.138218:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176966.138222:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176966.138224:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176966.138225:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301861898
00000100:00000001:1.0:1786176966.138227:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301865677
00000100:00000001:1.0:1786176966.138229:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301861898
00000100:00000001:1.0:1786176966.138230:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301865677
00000100:00000001:1.0:1786176966.138232:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176966.138233:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.138234:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176966.138237:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176966/real 1786176966]  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176966.138241:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176966.138242:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176966.138244:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176966.138245:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176966.138246:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176966.138248:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176966.138250:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176966.138255:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138259:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176966.138259:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138260:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176966.138261:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176966.138263:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138267:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176966.138270:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138275:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176966.138276:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138277:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176966.138278:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138279:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176966.138280:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176966.138282:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176966.138283:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176966.138285:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176966.138287:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176966.138292:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176966.138294:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138296:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176966.138298:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856320000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138302:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176966.138303:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176966.138304:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176966.138306:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176966.138307:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176966.138308:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176966.138309:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176966.138310:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3400.
02000000:00000001:1.0:1786176966.138312:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176966.138313:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176966.138314:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:1.0:1786176966.138316:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176966.138317:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138318:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176966.138320:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856320128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138325:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176966.138325:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176966.138326:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176966.138329:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176966.138330:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176966.138330:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176966.138331:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176966.138332:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:1.0:1786176966.138333:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176966.138334:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176966.138335:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:1.0:1786176966.138336:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176966.138337:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138338:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176966.138340:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856320256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138343:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176966.138344:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176966.138345:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176966.138346:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176966.138346:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176966.138347:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176966.138348:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176966.138349:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:1.0:1786176966.138350:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176966.138351:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176966.138352:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:1.0:1786176966.138353:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176966.138353:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138354:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138357:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176966.138358:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.138359:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176966.138360:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176966.138361:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176966.138363:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138368:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176966.138369:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.138370:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176966.138371:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176966.138373:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176966.138374:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176966.138376:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176966.138377:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176966.138378:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176966.138379:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176966.138381:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176966.138383:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176966.138385:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176966.138386:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176966.138388:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176966.138391:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856320384, portal 4
00000100:00000001:1.0:1786176966.138392:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176966.138393:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00000040:1.0:1786176966.138397:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138401:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176966.138403:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176966.138404:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176966.138406:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856320384, offset 0
00000400:00000200:1.0:1786176966.138408:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176966.138410:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176966.138413:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176966.138414:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176966.138416:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176966.138418:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176966.138420:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176966.138422:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254674:254674:256:4294967295] 192.168.202.110@tcp LPNI seq info [254663:254663:8:4294967295]
00000400:00000200:1.0:1786176966.138424:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176966.138429:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176966.138432:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176966.138433:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176966.138436:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176966.138438:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786176966.138441:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176966.138443:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786176966.138445:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176966.138446:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176966.138448:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176966.138451:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176966.138452:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176966.138454:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176966.138455:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176966.138457:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138462:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176966.138464:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176991 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138468:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.138469:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176966.138471:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176966.138473:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176966.138476:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176966.138477:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301861906
00000100:00000001:1.0:1786176966.138480:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301865677
00000100:00000001:1.0:1786176966.138482:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301861906
00000100:00000001:1.0:1786176966.138486:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301865677
00000100:00000001:1.0:1786176966.138488:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176966.138490:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.138492:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176966.138496:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176966/real 1786176966]  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176966.138505:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176966.138506:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176966.138508:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176966.138530:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176966.138532:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176966.138534:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176966.138537:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176966.138543:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138547:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176966.138548:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138549:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176966.138550:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176966.138553:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138557:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176966.138560:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138564:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176966.138565:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138566:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176966.138567:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138569:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176966.138570:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176966.138572:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176966.138574:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176966.138575:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176966.138577:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176966.138582:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176966.138584:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138586:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176966.138587:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856320384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176991 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176966.138591:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176966.138592:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176966.138594:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176966.138595:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176966.138596:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176966.138597:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176966.138598:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176966.138600:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:1.0:1786176966.138601:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176966.138602:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176966.138603:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:1.0:1786176966.138605:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176966.138605:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138607:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176966.138609:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176966.138610:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176966.138611:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176966.138612:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176966.712604:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176966.712611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176966.712617:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176966.712619:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176966.712623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176966.712624:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176966.712690:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176966.712693:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176966.712696:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176967.160580:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176967.160587:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176967.160588:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176967.160593:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176967.160595:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176967.160599:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176967.160601:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176967.160603:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176967.160605:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176967.736648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176967.736655:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176967.736661:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176967.736662:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176967.736664:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176967.736665:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176967.736733:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176967.736738:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176967.736742:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176968.184676:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176968.184686:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176968.184689:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176968.184697:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176968.184702:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176968.184706:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176968.184710:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176968.184713:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176968.184717:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176968.760618:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176968.760629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176968.760637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176968.760639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176968.760642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176968.760644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176968.760701:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176968.760707:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176968.760711:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176969.208574:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176969.208580:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176969.208582:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176969.208587:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176969.208590:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176969.208593:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176969.208596:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176969.208598:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176969.208600:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176969.784601:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176969.784606:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176969.784611:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176969.784611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176969.784614:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176969.784615:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176969.784667:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176969.784670:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176969.784673:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176969.976605:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301865677, at 4301865728
00000100:00000040:1.0:1786176969.976605:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301865677, at 4301865728
00000100:00000010:1.0:1786176969.976621:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b11e0.
00000100:00000010:0.0:1786176969.976624:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1f00.
00000100:00000001:1.0:1786176970.232615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176970.232622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176970.232624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176970.232628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176970.232631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176970.232635:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176970.232637:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176970.232640:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176970.232642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176970.808628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176970.808634:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176970.808636:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176970.808638:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176970.808642:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176970.808644:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176970.808646:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176970.808648:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176970.808651:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176970.808652:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176970.808653:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176970.808654:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176970.808655:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176970.808656:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176970.808657:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176970.808659:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176970.808661:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176970.808662:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176970.808724:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176970.808729:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176970.808733:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176971.256622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1786176971.256622:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176971.256628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176971.256632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176971.256632:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786176971.256638:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:1.0:1786176971.256639:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:3.0:1786176971.256642:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176971.256643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176971.256648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176971.256648:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176971.256652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176971.256655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176971.256656:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7194
00000100:00000001:1.0:1786176971.256659:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176971.256666:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176971.256672:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176971.256675:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176971.256678:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.256681:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176971.256683:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.256685:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176971.256688:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176971.256689:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176971.256691:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176971.256694:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176971.256695:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176971.256697:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176971.256699:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.256700:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176971.256701:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.256703:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176971.256705:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176971.256708:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176971.256709:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176971.256711:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176971.256713:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176971.256715:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.256717:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176971.256718:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176971.256726:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69dc00.
00000020:00000040:3.0:1786176971.256728:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176971.256731:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176971.256732:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.256734:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176971.256737:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176971.256740:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176971.256750:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000001:3.0:1786176971.256757:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176971.256766:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176971.256776:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176971.256781:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176971.256785:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097fe80e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176971.256809:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176971.256812:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176971.256816:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176971.256818:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176971.256820:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176971.256823:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176971.256826:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7194
00000400:00000200:3.0:1786176971.256830:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176971.256833:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176971.256836:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176971.256838:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.256840:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176971.256841:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.256842:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176971.256844:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176971.256845:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786176971.256846:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176971.256848:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786176971.256849:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176971.256851:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786176971.256853:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.256856:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176971.256857:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.256859:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176971.256861:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786176971.256863:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176971.256864:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176971.256866:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176971.256867:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:3.0:1786176971.256870:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176971.256872:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
02000000:00000001:3.0:1786176971.256873:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.256877:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176971.256878:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000008:00100000:3.0:1786176971.256880:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176971.256884:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.256886:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176971.256888:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1786176971.256891:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e680.
00000020:00000040:3.0:1786176971.256895:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176971.256898:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176971.256900:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.256904:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176971.256907:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176971.256908:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1786176971.256909:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.256910:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176971.256915:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:3.0:1786176971.256915:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176971.256919:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176971.256920:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176971.256922:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176971.256928:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:1.0:1786176971.256929:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856320512:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176971.256937:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1786176971.256941:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1786176971.256942:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176971.256958:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176971.256961:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176971.256967:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
00000100:00000001:3.0:1786176971.256967:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176971.256971:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176971.256976:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1839131] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176971.256980:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176971.256987:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000100:00000001:3.0:1786176971.256988:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176971.256993:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176971.256996:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:1.0:1786176971.256998:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000001:3.0:1786176971.256998:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176971.257001:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176971.257004:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176971.257006:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7194
00000100:00000200:1.0:1786176971.257008:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856320512, portal 25
00000400:00000200:3.0:1786176971.257009:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176971.257014:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176971.257015:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1786176971.257017:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176971.257019:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764299776 : -114547945251840 : ffff97d1ba69dc00)
00000100:00000040:3.0:1786176971.257020:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.257023:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176971.257027:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.257028:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176971.257030:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176971.257032:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000100:00000040:1.0:1786176971.257034:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786176971.257034:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176971.257037:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786176971.257040:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786176971.257044:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786176971.257050:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176971.257054:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786176971.257054:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176971.257059:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176971.257062:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176971.257063:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000010:1.0:1786176971.257067:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000020:00000040:3.0:1786176971.257067:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786176971.257071:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:1.0:1786176971.257073:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856320512, offset 0
00000100:00000001:3.0:1786176971.257074:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176971.257077:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176971.257080:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786176971.257084:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786176971.257084:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.257088:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176971.257090:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176971.257091:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.257093:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176971.257094:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:3.0:1786176971.257098:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ed80.
00000020:00000040:3.0:1786176971.257101:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:1.0:1786176971.257103:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:3.0:1786176971.257104:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176971.257107:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.257110:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1786176971.257111:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1786176971.257112:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176971.257115:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176971.257117:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000010:3.0:1786176971.257120:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000400:00000200:1.0:1786176971.257122:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1786176971.257125:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176971.257127:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00100000:3.0:1786176971.257132:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176971.257133:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254675:254675:256:4294967295] 192.168.202.110@tcp LPNI seq info [254664:254664:8:4294967295]
00000400:00000200:1.0:1786176971.257145:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176971.257151:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176971.257153:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176971.257161:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1786176971.257161:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ccd6c6d2] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1786176971.257172:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176971.257177:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786176971.257180:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000100:00080000:3.0:1786176971.257180:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176971.257184:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786176971.257185:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176971.257187:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176971.257189:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176971.257192:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7194
00000400:00000200:3.0:1786176971.257195:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176971.257200:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:1.0:1786176971.257203:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786176971.257203:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176971.257208:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:1.0:1786176971.257212:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:3.0:1786176971.257212:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176971.257223:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:1.1:1786176971.257224:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786176971.257226:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176971.257230:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176971.257232:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000800:00000200:1.1:1786176971.257233:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:3.0:1786176971.257234:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176971.257236:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786176971.257240:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000100:1.0:1786176971.257244:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:3.0:1786176971.257244:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000010:1.0:1786176971.257249:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000100:00000040:3.0:1786176971.257249:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.257253:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176971.257255:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176971.257257:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1786176971.257257:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176971.257261:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786176971.257267:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1786176971.257268:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1786176971.257272:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176971.257274:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786176971.257275:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
02000000:00000001:3.0:1786176971.257276:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:1.0:1786176971.257280:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
02000000:00000001:3.0:1786176971.257280:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.257285:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176971.257287:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000008:00100000:3.0:1786176971.257288:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176971.257291:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.257293:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176971.257295:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1786176971.257297:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c380.
00000020:00000040:3.0:1786176971.257301:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176971.257305:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176971.257307:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.257310:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786176971.257312:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1786176971.257312:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176971.257313:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176971.257318:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000040:1.0:1786176971.257319:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176971.257321:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176971.257329:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176971.257330:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.257334:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176971.257340:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176971.257341:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.257342:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176971.257348:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1786176971.257349:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000833bf1c9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257352:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301870797
00000100:00000010:1.0:1786176971.257359:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b11e0.
00000100:00000001:3.0:1786176971.257365:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.257366:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:3.0:1786176971.257368:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176971.257371:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1786176971.257373:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301870797
00000100:00000001:3.0:1786176971.257374:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176971.257376:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000010:1.0:1786176971.257378:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1600.
00000100:00000001:3.0:1786176971.257380:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176971.257382:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00080000:3.0:1786176971.257384:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7194
00000100:00000001:1.0:1786176971.257387:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786176971.257389:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176971.257391:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176971.257394:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176971.257396:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00080000:3.0:1786176971.257397:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00100000:1.0:1786176971.257402:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176971/real 1786176971]  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176971.257403:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.257406:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176971.257411:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.257414:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176971.257419:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176971.257421:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000400:00000001:1.0:1786176971.257422:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:3.0:1786176971.257423:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000001:1.0:1786176971.257426:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:3.0:1786176971.257426:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786176971.257430:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000010:1.0:1786176971.257431:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000020:00000001:3.0:1786176971.257434:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:1.0:1786176971.257435:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000100:00000040:3.0:1786176971.257439:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:1.0:1786176971.257440:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:3.0:1786176971.257442:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176971.257445:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:3.0:1786176971.257446:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176971.257448:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:1.0:1786176971.257450:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786176971.257451:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786176971.257457:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176971.257462:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176971.257466:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176971.257469:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1786176971.257471:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176971.257472:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.257476:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176971.257479:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176971.257483:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.257485:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176971.257486:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.257488:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:3.0:1786176971.257490:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d880.
00000100:00000001:1.0:1786176971.257491:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176971.257495:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:3.0:1786176971.257495:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176971.257499:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:1.0:1786176971.257500:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176971.257501:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176971.257505:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176971.257507:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176971.257530:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786176971.257535:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176971.257536:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257544:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176971.257546:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.257548:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000010:3.0:1786176971.257548:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:1.0:1786176971.257550:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176971.257553:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.257554:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176971.257558:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00100000:3.0:1786176971.257562:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00080000:1.0:1786176971.257563:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176971.257570:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176971.257574:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1786176971.257579:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1786176971.257583:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176971.257583:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176971.257591:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097f9b8db] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176971.257598:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856320512:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786176971.257604:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.257606:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.257609:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786176971.257609:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7204)
00000100:00000040:1.0:1786176971.257616:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69dc00 x1872934856320512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257621:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176971.257623:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176971.257625:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176971.257627:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176971.257629:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176971.257630:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176971.257632:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176971.257633:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786176971.257637:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176971.257638:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176971.257640:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69dc00.
00000100:00000001:1.0:1786176971.257642:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176971.257643:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.257644:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.257649:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176971.257650:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.257651:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176971.257653:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176971.257654:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176971.257656:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257661:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176971.257662:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176971.257663:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176971.257665:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176971.257666:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176971.257668:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176971.257670:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176971.257671:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176971.257672:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176971.257673:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176971.257675:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176971.257676:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176971.257678:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176971.257680:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176971.257682:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176971.257684:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856320640, portal 10
00000100:00000001:1.0:1786176971.257686:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176971.257687:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00000040:1.0:1786176971.257691:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257696:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176971.257697:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176971.257699:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176971.257700:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856320640, offset 0
00000400:00000200:1.0:1786176971.257703:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176971.257706:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176971.257708:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176971.257711:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176971.257712:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176971.257714:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176971.257716:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176971.257718:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254676:254676:256:4294967295] 192.168.202.110@tcp LPNI seq info [254665:254665:8:4294967295]
00000400:00000200:1.0:1786176971.257721:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176971.257725:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176971.257728:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176971.257730:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176971.257733:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176971.257735:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786176971.257738:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176971.257739:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1786176971.257742:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176971.257743:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176971.257745:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176971.257748:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176971.257749:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176971.257751:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176971.257752:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176971.257755:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257760:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176971.257761:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257766:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.257767:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176971.257769:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176971.257773:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176971.257775:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176971.257776:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301867018
00000100:00000001:1.0:1786176971.257778:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301870797
00000100:00000001:1.0:1786176971.257780:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301867018
00000100:00000001:1.0:1786176971.257781:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301870797
00000100:00000001:1.0:1786176971.257782:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176971.257783:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.257784:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176971.257787:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176971/real 1786176971]  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176971.257792:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176971.257793:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176971.257794:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176971.257795:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176971.257796:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176971.257798:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176971.257800:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176971.257805:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257809:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176971.257810:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.257811:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176971.257812:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176971.257814:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257818:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176971.257821:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257827:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176971.257830:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.257832:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176971.257833:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.257835:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176971.257837:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176971.257840:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176971.257843:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176971.257845:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176971.257850:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176971.257858:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176971.257862:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176971.257865:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257873:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176971.257874:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176971.257876:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176971.257878:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176971.257881:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176971.257883:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176971.257885:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176971.257887:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176971.257888:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176971.257889:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176971.257892:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176971.257893:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176971.257896:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176971.257897:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176971.257899:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176971.257902:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856320768, portal 10
00000100:00000001:1.0:1786176971.257903:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176971.257904:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:1.0:1786176971.257908:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257912:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176971.257914:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176971.257915:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176971.257917:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856320768, offset 0
00000400:00000200:1.0:1786176971.257919:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176971.257921:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176971.257923:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176971.257925:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176971.257927:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176971.257929:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176971.257930:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176971.257932:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254677:254677:256:4294967295] 192.168.202.110@tcp LPNI seq info [254666:254666:8:4294967295]
00000400:00000200:1.0:1786176971.257935:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176971.257939:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176971.257942:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176971.257957:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176971.257961:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176971.257962:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786176971.257965:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176971.257967:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1786176971.257970:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176971.257971:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176971.257973:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176971.257976:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176971.257978:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176971.257980:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176971.257983:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176971.257986:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.257994:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176971.257997:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258004:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.258006:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176971.258010:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176971.258013:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176971.258016:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176971.258018:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301867143
00000100:00000001:1.0:1786176971.258021:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301870797
00000100:00000001:1.0:1786176971.258023:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301867143
00000100:00000001:1.0:1786176971.258025:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301870797
00000100:00000001:1.0:1786176971.258028:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176971.258030:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.258032:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176971.258036:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176971/real 1786176971]  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176971.258042:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176971.258043:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176971.258044:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176971.258046:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176971.258047:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176971.258048:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176971.258051:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176971.258056:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258061:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176971.258062:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258063:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176971.258064:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176971.258066:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258070:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176971.258073:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258077:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176971.258079:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258080:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176971.258081:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258082:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176971.258083:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176971.258085:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176971.258086:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176971.258088:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176971.258090:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176971.258095:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176971.258097:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176971.258099:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258103:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176971.258104:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176971.258106:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176971.258108:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176971.258109:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176971.258111:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176971.258113:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176971.258114:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176971.258115:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176971.258116:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176971.258119:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176971.258120:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176971.258122:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176971.258124:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176971.258126:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176971.258128:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856320896, portal 4
00000100:00000001:1.0:1786176971.258130:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176971.258131:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764293504 : -114547945258112 : ffff97d1ba69c380)
00000100:00000040:1.0:1786176971.258134:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258139:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176971.258140:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176971.258141:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176971.258143:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856320896, offset 0
00000400:00000200:1.0:1786176971.258146:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176971.258148:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176971.258150:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176971.258152:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176971.258154:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176971.258155:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176971.258157:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176971.258159:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254678:254678:256:4294967295] 192.168.202.110@tcp LPNI seq info [254667:254667:8:4294967295]
00000400:00000200:1.0:1786176971.258162:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176971.258165:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176971.258169:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176971.258171:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176971.258174:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176971.258176:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786176971.258178:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176971.258180:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1786176971.258183:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176971.258184:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176971.258187:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176971.258191:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176971.258194:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176971.258195:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176971.258198:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176971.258201:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258209:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176971.258213:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258220:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.258221:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176971.258225:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176971.258229:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176971.258232:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176971.258234:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301867018
00000100:00000001:1.0:1786176971.258237:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301870797
00000100:00000001:1.0:1786176971.258239:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301867018
00000100:00000001:1.0:1786176971.258240:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301870797
00000100:00000001:1.0:1786176971.258242:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176971.258243:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.258245:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176971.258247:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176971/real 1786176971]  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176971.258253:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176971.258254:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176971.258256:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176971.258257:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176971.258258:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176971.258260:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176971.258262:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176971.258268:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258272:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176971.258273:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258274:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176971.258275:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176971.258277:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258281:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176971.258283:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258287:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176971.258288:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258289:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176971.258290:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258292:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176971.258293:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176971.258294:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176971.258296:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176971.258297:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176971.258300:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176971.258305:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856320896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176971.258308:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176971.258310:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258314:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176971.258315:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176971.258316:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176971.258317:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176971.258318:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176971.258320:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856321024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176971.258322:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176971.258323:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176971.258324:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176971.258325:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176971.258327:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176971.258328:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176971.258330:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176971.258332:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176971.258334:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176971.258336:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856321024, portal 4
00000100:00000001:1.0:1786176971.258337:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176971.258338:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
00000100:00000040:1.0:1786176971.258342:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258346:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176971.258348:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176971.258349:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176971.258350:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856321024, offset 0
00000400:00000200:1.0:1786176971.258354:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176971.258356:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176971.258358:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176971.258360:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176971.258362:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176971.258364:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176971.258365:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176971.258367:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254679:254679:256:4294967295] 192.168.202.110@tcp LPNI seq info [254668:254668:8:4294967295]
00000400:00000200:1.0:1786176971.258370:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176971.258374:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176971.258377:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176971.258378:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176971.258381:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176971.258383:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786176971.258385:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176971.258387:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1786176971.258389:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176971.258391:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176971.258393:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176971.258395:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176971.258397:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176971.258398:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176971.258399:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176971.258402:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258407:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176971.258408:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258413:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.258414:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176971.258415:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176971.258417:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176971.258419:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176971.258420:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301867026
00000100:00000001:1.0:1786176971.258422:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301870797
00000100:00000001:1.0:1786176971.258424:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301867026
00000100:00000001:1.0:1786176971.258425:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301870797
00000100:00000001:1.0:1786176971.258426:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176971.258427:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.258428:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176971.258430:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176971/real 1786176971]  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176971.258435:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176971.258435:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176971.258436:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176971.258438:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176971.258439:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176971.258440:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176971.258442:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176971.258447:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258451:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176971.258452:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258453:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176971.258454:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176971.258456:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258460:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176971.258462:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258466:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176971.258468:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258469:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176971.258470:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258471:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176971.258472:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176971.258474:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176971.258475:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176971.258477:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176971.258479:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176971.258483:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856321024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176971.258485:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258487:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176971.258489:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856320640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258492:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176971.258493:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176971.258495:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176971.258496:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176971.258497:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176971.258498:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176971.258499:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176971.258500:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176971.258502:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176971.258503:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176971.258504:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:1.0:1786176971.258506:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176971.258507:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258508:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176971.258526:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856320768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258535:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176971.258536:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176971.258537:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176971.258540:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176971.258540:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176971.258541:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176971.258542:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176971.258543:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:1.0:1786176971.258545:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176971.258546:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176971.258546:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:1.0:1786176971.258548:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176971.258549:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258550:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176971.258552:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c380 x1872934856320896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258556:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176971.258556:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176971.258557:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176971.258558:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176971.258559:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176971.258560:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176971.258561:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176971.258562:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:1.0:1786176971.258563:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176971.258563:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176971.258564:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c380.
00000100:00000001:1.0:1786176971.258565:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176971.258566:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258567:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176971.258569:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856321024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176971.258573:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176971.258574:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176971.258576:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176971.258577:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176971.258578:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176971.258579:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176971.258580:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176971.258581:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176971.258582:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176971.258583:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176971.258583:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:1.0:1786176971.258585:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176971.258585:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258586:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176971.258589:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176971.258590:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176971.258591:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176971.258592:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176971.832599:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176971.832605:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176971.832610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176971.832611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176971.832613:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176971.832614:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176971.832666:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176971.832670:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176971.832672:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176972.280601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176972.280607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176972.280609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176972.280613:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176972.280616:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176972.280620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176972.280621:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176972.280624:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176972.280626:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176972.856643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176972.856649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176972.856653:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176972.856654:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176972.856656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176972.856660:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176972.856661:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176972.856664:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176972.856666:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176972.856668:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176972.856670:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176972.856671:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176972.856762:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176972.856766:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176972.856769:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176973.304626:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176973.304631:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176973.304633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176973.304638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176973.304641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176973.304644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176973.304646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176973.304648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176973.304651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176973.880619:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176973.880626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176973.880631:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176973.880633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176973.880636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176973.880638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176973.880704:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176973.880709:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176973.880713:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176974.328597:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176974.328604:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176974.328606:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176974.328610:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176974.328612:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176974.328616:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176974.328618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176974.328620:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176974.328622:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176974.904629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176974.904634:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176974.904638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176974.904639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176974.904642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176974.904643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176974.904695:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176974.904699:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176974.904702:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176975.096573:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301870797, at 4301870848
00000100:00000010:0.0:1786176975.096586:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b11e0.
00000100:00000040:1.0:1786176975.096590:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301870797, at 4301870848
00000100:00000010:1.0:1786176975.096600:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1600.
00000100:00000001:1.0:1786176975.352623:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176975.352629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176975.352631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176975.352636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176975.352638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176975.352642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176975.352644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176975.352646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176975.352648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176975.928628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176975.928638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176975.928653:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176975.928654:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176975.928658:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176975.928659:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176975.928733:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176975.928738:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176975.928741:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176976.376644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176976.376653:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786176976.376655:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176976.376656:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176976.376664:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:0.0:1786176976.376666:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786176976.376671:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176976.376677:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176976.376677:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176976.376682:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176976.376685:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176976.376686:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176976.376690:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.376693:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176976.376697:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7199
00000400:00000200:0.0:1786176976.376708:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786176976.376716:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786176976.376720:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176976.376725:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176976.376730:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176976.376733:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176976.376736:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176976.376740:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176976.376743:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786176976.376746:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176976.376749:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786176976.376752:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786176976.376755:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786176976.376758:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176976.376760:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176976.376762:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176976.376764:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176976.376767:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786176976.376771:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176976.376774:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176976.376778:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786176976.376782:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786176976.376786:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176976.376788:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786176976.376791:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176976.376803:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418700.
00000020:00000040:0.0:1786176976.376808:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786176976.376812:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176976.376815:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176976.376818:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176976.376822:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176976.376827:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786176976.376837:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000100:00000001:0.0:1786176976.376845:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786176976.376858:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786176976.376870:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786176976.376877:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176976.376882:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bf66f89d] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176976.376937:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176976.376942:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176976.376946:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786176976.376948:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786176976.376950:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176976.376954:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786176976.376958:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7199
00000400:00000200:0.0:1786176976.376963:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176976.376965:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1786176976.376969:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176976.376970:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176976.376974:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1786176976.376975:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786176976.376983:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176976.376983:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176976.376990:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176976.376995:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176976.376997:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786176976.376999:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176976.377000:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176976.377006:0:62275:0:(genops.c:813:class_conn2export()) Process entered
02000000:00000001:1.0:1786176976.377007:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1786176976.377010:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
02000000:00000001:1.0:1786176976.377011:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1786176976.377015:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
02000000:00000001:1.0:1786176976.377016:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786176976.377020:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00100000:1.0:1786176976.377022:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856321152:192.168.202.110@tcp:250:kworker.0
00000020:00000001:0.0:1786176976.377027:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176976.377028:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:0.0:1786176976.377033:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:1.0:1786176976.377035:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176976.377039:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176976.377042:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176976.377043:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1786176976.377048:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176976.377050:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000010:1.0:1786176976.377052:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
00000100:00000001:0.0:1786176976.377054:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:1.0:1786176976.377055:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786176976.377061:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000400:00000200:1.0:1786176976.377065:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786176976.377067:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:1.0:1786176976.377070:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000100:00000001:0.0:1786176976.377072:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176976.377076:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:1.0:1786176976.377079:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
02000000:00000001:0.0:1786176976.377084:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1786176976.377088:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856321152, portal 25
02000000:00000001:0.0:1786176976.377089:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.377094:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786176976.377095:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176976.377097:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761656576 : -114547947895040 : ffff97d1ba418700)
00000008:00100000:0.0:1786176976.377103:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000040:1.0:1786176976.377104:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1786176976.377111:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.377116:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1786176976.377117:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176976.377120:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000010:0.0:1786176976.377125:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419f80.
00000400:00000010:1.0:1786176976.377126:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176976.377130:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856321152, offset 0
00000020:00000040:0.0:1786176976.377132:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786176976.377138:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:1.0:1786176976.377139:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786176976.377142:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176976.377146:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176976.377148:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176976.377150:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176976.377151:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1786176976.377160:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4000.
00000400:00000200:1.0:1786176976.377161:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1786176976.377169:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176976.377170:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176976.377179:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00100000:0.0:1786176976.377181:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176976.377185:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176976.377190:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176976.377198:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254680:254680:256:4294967295] 192.168.202.110@tcp LPNI seq info [254669:254669:8:4294967295]
00000100:00000001:0.0:1786176976.377202:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176976.377203:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176976.377207:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176976.377216:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ad753ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176976.377217:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176976.377229:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786176976.377237:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786176976.377238:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000100:00080000:0.0:1786176976.377241:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176976.377247:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000800:00000200:1.0:1786176976.377249:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786176976.377251:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:1.0:1786176976.377255:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00080000:0.0:1786176976.377256:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786176976.377262:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.1:1786176976.377263:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176976.377268:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00080000:0.0:1786176976.377271:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7199
00000800:00000100:1.0:1786176976.377275:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176976.377279:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:0.0:1786176976.377282:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176976.377287:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176976.377289:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1786176976.377295:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:0.0:1786176976.377296:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786176976.377300:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000100:00000040:0.0:1786176976.377304:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:1.0:1786176976.377304:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:0.0:1786176976.377310:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176976.377311:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:0.0:1786176976.377316:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:1.0:1786176976.377317:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176976.377322:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176976.377329:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176976.377332:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1786176976.377333:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000100:00000040:1.0:1786176976.377337:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786176976.377339:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786176976.377343:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000001:1.0:1786176976.377349:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176976.377350:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:1.0:1786176976.377352:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:0.0:1786176976.377356:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:1.0:1786176976.377361:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176976.377364:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176976.377368:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176976.377370:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176976.377372:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301875917
00000100:00000040:0.0:1786176976.377377:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000010:1.0:1786176976.377380:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1600.
00000100:00000001:0.0:1786176976.377381:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176976.377386:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000001:1.0:1786176976.377389:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:0.0:1786176976.377390:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786176976.377397:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.377398:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301875917
02000000:00000001:0.0:1786176976.377400:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:1.0:1786176976.377402:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1900.
02000000:00000001:0.0:1786176976.377406:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176976.377406:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:0.0:1786176976.377412:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.377412:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176976.377416:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176976.377419:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176976.377420:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000008:00100000:0.0:1786176976.377423:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00100000:1.0:1786176976.377426:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176976/real 1786176976]  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1786176976.377428:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176976.377433:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176976.377441:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418000.
00000400:00000001:1.0:1786176976.377441:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176976.377445:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:0.0:1786176976.377447:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000010:1.0:1786176976.377448:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176976.377452:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
02000000:00000001:0.0:1786176976.377453:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:1.0:1786176976.377455:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
02000000:00000001:0.0:1786176976.377457:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.377458:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176976.377462:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176976.377468:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176976.377473:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786176976.377476:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:0.0:1786176976.377478:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176976.377479:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786176976.377483:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.377483:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176976.377485:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176976.377490:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786176976.377492:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000100:00000001:1.0:1786176976.377499:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176976.377500:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176976.377505:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.377532:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176976.377535:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.377537:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176976.377538:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.377540:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176976.377542:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00100000:0.0:1786176976.377544:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00080000:1.0:1786176976.377545:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176976.377553:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176976.377559:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1786176976.377566:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1786176976.377568:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176976.377571:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786176976.377582:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002e10e4f5] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176976.377582:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856321152:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176976.377590:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.377594:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176976.377599:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418700 x1872934856321152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176976.377603:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786176976.377609:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786176976.377619:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1786176976.377621:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000001:0.0:1786176976.377622:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:1.0:1786176976.377624:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00080000:0.0:1786176976.377626:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
02000000:00000010:1.0:1786176976.377628:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
00000100:00000001:0.0:1786176976.377630:0:62275:0:(import.c:485:import_select_connection()) Process entered
02000000:00000001:1.0:1786176976.377632:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00080000:0.0:1786176976.377637:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7199
00000020:00000001:1.0:1786176976.377637:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176976.377640:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176976.377643:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000400:00000200:0.0:1786176976.377647:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:1.0:1786176976.377647:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:1.0:1786176976.377654:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000400:00000200:0.0:1786176976.377656:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:1.0:1786176976.377657:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176976.377660:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418700.
00000100:00080000:0.0:1786176976.377662:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176976.377664:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176976.377667:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786176976.377669:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176976.377670:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786176976.377675:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786176976.377678:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786176976.377681:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176976.377681:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176976.377684:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176976.377686:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176976.377687:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1786176976.377689:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:1.0:1786176976.377690:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1786176976.377694:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176976.377694:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1786176976.377699:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000040:1.0:1786176976.377700:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786176976.377703:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786176976.377709:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176976.377717:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1786176976.377719:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176976.377720:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176976.377724:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1786176976.377726:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1786176976.377729:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176976.377731:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176976.377733:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:1.0:1786176976.377737:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786176976.377738:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786176976.377747:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:1.0:1786176976.377747:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856321280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176976.377753:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.377756:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786176976.377757:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786176976.377761:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786176976.377762:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176976.377765:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786176976.377767:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176976.377770:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1786176976.377772:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786176976.377775:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786176976.377778:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176976.377782:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176976.377783:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176976.377786:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786176976.377787:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b800.
00000400:00000200:1.0:1786176976.377791:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1786176976.377792:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000400:00000010:1.0:1786176976.377797:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
02000000:00000001:0.0:1786176976.377798:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786176976.377801:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176976.377803:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000001:0.0:1786176976.377805:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:1.0:1786176976.377808:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856321280, portal 10
02000000:00000001:0.0:1786176976.377809:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176976.377813:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1786176976.377814:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.377815:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761662848 : -114547947888768 : ffff97d1ba419f80)
02000000:00000010:0.0:1786176976.377823:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000040:1.0:1786176976.377825:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176976.377829:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.377834:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176976.377841:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00100000:0.0:1786176976.377842:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786176976.377844:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176976.377848:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856321280, offset 0
00000400:00000200:1.0:1786176976.377857:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176976.377863:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176976.377864:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176976.377868:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1786176976.377870:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176976.377873:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176976.377877:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:0.0:1786176976.377882:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000192ee9a0] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176976.377882:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176976.377886:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176976.377891:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254681:254681:256:4294967295] 192.168.202.110@tcp LPNI seq info [254670:254670:8:4294967295]
00000400:00000200:1.0:1786176976.377916:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786176976.377920:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176976.377926:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1786176976.377928:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.0:1786176976.377933:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786176976.377937:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000010:1.0:1786176976.377938:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000100:00000001:0.0:1786176976.377941:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:1.0:1786176976.377945:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1786176976.377947:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:1.0:1786176976.377949:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1786176976.377953:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.1:1786176976.377955:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176976.377958:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00080000:0.0:1786176976.377960:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7199
00000800:00000100:1.0:1786176976.377965:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176976.377967:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:0.0:1786176976.377972:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176976.377972:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786176976.377983:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1786176976.377983:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176976.377988:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000100:00080000:0.0:1786176976.377990:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:1.0:1786176976.377991:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176976.377997:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:0.0:1786176976.378000:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:1.0:1786176976.378002:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176976.378006:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176976.378012:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786176976.378013:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176976.378018:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176976.378019:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786176976.378025:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786176976.378029:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:1.0:1786176976.378030:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176976.378034:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176976.378034:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176976.378038:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786176976.378039:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000040:1.0:1786176976.378043:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000020:00000001:0.0:1786176976.378044:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:1.0:1786176976.378050:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:0.0:1786176976.378051:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176976.378052:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301872138
00000100:00000001:1.0:1786176976.378057:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301875917
00000100:00000001:1.0:1786176976.378060:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301872138
00000100:00000040:0.0:1786176976.378061:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176976.378062:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301875917
00000100:00000001:1.0:1786176976.378066:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176976.378068:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176976.378069:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.378072:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:0.0:1786176976.378074:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00100000:1.0:1786176976.378078:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176976/real 1786176976]  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176976.378080:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786176976.378087:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000400:00000001:1.0:1786176976.378091:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176976.378094:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:0.0:1786176976.378096:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:1.0:1786176976.378097:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176976.378100:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000100:00000001:0.0:1786176976.378102:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176976.378103:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
02000000:00000001:0.0:1786176976.378108:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176976.378108:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:0.0:1786176976.378114:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1786176976.378114:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786176976.378121:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176976.378127:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000200:1.0:1786176976.378127:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1786176976.378132:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786176976.378139:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786176976.378144:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.378148:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000010:0.0:1786176976.378151:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418380.
00000100:00000001:1.0:1786176976.378151:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378152:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176976.378154:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1786176976.378157:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00100000:1.0:1786176976.378159:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786176976.378164:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176976.378167:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786176976.378168:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176976.378172:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176976.378173:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786176976.378178:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786176976.378181:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.378185:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000010:0.0:1786176976.378188:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000100:00000001:1.0:1786176976.378188:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378189:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176976.378191:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378194:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786176976.378196:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.378196:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176976.378203:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176976.378207:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00100000:0.0:1786176976.378208:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176976.378209:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176976.378216:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786176976.378232:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00100000:1.0:1786176976.378235:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856321280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786176976.378238:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.378240:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176976.378249:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786176976.378250:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e6b901d7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378261:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176976.378264:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176976.378268:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176976.378270:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786176976.378273:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176976.378273:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176976.378279:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856321408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176976.378281:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:0.0:1786176976.378282:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7209)
02000000:00000001:1.0:1786176976.378283:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176976.378285:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176976.378288:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176976.378291:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176976.378293:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176976.378296:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176976.378298:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176976.378301:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176976.378304:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856321408, portal 10
00000100:00000001:1.0:1786176976.378305:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176976.378306:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761654784 : -114547947896832 : ffff97d1ba418000)
00000100:00000040:1.0:1786176976.378311:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378316:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176976.378318:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176976.378319:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176976.378321:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856321408, offset 0
00000400:00000200:1.0:1786176976.378323:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176976.378326:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176976.378329:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176976.378331:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176976.378333:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176976.378335:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176976.378337:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176976.378339:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254682:254682:256:4294967295] 192.168.202.110@tcp LPNI seq info [254671:254671:8:4294967295]
00000400:00000200:1.0:1786176976.378342:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176976.378346:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176976.378350:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176976.378352:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176976.378355:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176976.378357:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786176976.378360:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176976.378362:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1786176976.378364:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176976.378366:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176976.378368:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176976.378370:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176976.378372:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176976.378373:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176976.378375:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176976.378377:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378383:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176976.378385:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378389:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.378390:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176976.378392:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176976.378393:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176976.378395:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176976.378396:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301872263
00000100:00000001:1.0:1786176976.378398:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301875917
00000100:00000001:1.0:1786176976.378399:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301872263
00000100:00000001:1.0:1786176976.378401:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301875917
00000100:00000001:1.0:1786176976.378402:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176976.378404:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.378405:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176976.378407:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176976/real 1786176976]  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176976.378413:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176976.378414:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176976.378416:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176976.378417:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176976.378418:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176976.378419:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176976.378421:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176976.378426:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378431:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176976.378432:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378433:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176976.378434:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176976.378436:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378440:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176976.378442:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378446:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176976.378448:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378448:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176976.378449:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378450:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176976.378452:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176976.378453:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176976.378455:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176976.378456:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176976.378459:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176976.378465:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856321408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176976.378467:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378468:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176976.378470:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419f80 x1872934856321280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378474:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176976.378475:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176976.378476:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176976.378478:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176976.378479:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176976.378480:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176976.378481:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176976.378482:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4000.
02000000:00000001:1.0:1786176976.378483:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176976.378484:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176976.378485:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419f80.
00000100:00000001:1.0:1786176976.378487:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176976.378488:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378489:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176976.378491:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418000 x1872934856321408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378496:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176976.378496:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176976.378497:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176976.378499:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176976.378499:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176976.378500:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176976.378501:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176976.378502:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:1.0:1786176976.378503:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176976.378504:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176976.378505:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418000.
00000100:00000001:1.0:1786176976.378507:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176976.378508:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378525:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378528:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176976.378529:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.378531:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176976.378532:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176976.378533:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176976.378535:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378541:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176976.378542:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176976.378543:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176976.378545:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176976.378546:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176976.378548:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856321536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176976.378550:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176976.378551:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176976.378553:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176976.378554:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176976.378556:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176976.378557:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176976.378560:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176976.378561:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176976.378564:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176976.378566:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856321536, portal 4
00000100:00000001:1.0:1786176976.378567:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176976.378568:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761669120 : -114547947882496 : ffff97d1ba41b800)
00000100:00000040:1.0:1786176976.378573:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378577:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176976.378579:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176976.378581:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176976.378583:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856321536, offset 0
00000400:00000200:1.0:1786176976.378585:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176976.378588:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176976.378590:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176976.378593:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176976.378594:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176976.378596:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176976.378598:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176976.378600:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254683:254683:256:4294967295] 192.168.202.110@tcp LPNI seq info [254672:254672:8:4294967295]
00000400:00000200:1.0:1786176976.378603:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176976.378607:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176976.378610:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176976.378611:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176976.378614:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176976.378616:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786176976.378619:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176976.378621:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1786176976.378623:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176976.378625:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176976.378627:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176976.378629:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176976.378631:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176976.378632:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176976.378634:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176976.378637:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378642:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176976.378644:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378649:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.378650:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176976.378652:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176976.378654:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176976.378657:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176976.378658:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301872139
00000100:00000001:1.0:1786176976.378660:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301875917
00000100:00000001:1.0:1786176976.378661:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301872139
00000100:00000001:1.0:1786176976.378663:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301875917
00000100:00000001:1.0:1786176976.378664:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176976.378665:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.378666:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176976.378668:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176976/real 1786176976]  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176976.378673:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176976.378674:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176976.378675:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176976.378677:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176976.378678:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176976.378679:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176976.378681:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176976.378686:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378691:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176976.378692:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378693:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176976.378694:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176976.378696:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378700:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176976.378703:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378708:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176976.378711:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378712:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176976.378714:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378716:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176976.378718:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176976.378720:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176976.378724:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176976.378726:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176976.378730:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176976.378739:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856321536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176976.378744:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176976.378748:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378755:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176976.378757:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176976.378760:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176976.378761:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176976.378763:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176976.378767:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856321664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176976.378769:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176976.378771:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176976.378772:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176976.378773:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176976.378776:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176976.378777:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176976.378780:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176976.378782:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176976.378784:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176976.378787:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856321664, portal 4
00000100:00000001:1.0:1786176976.378788:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176976.378789:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761655680 : -114547947895936 : ffff97d1ba418380)
00000100:00000040:1.0:1786176976.378794:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378799:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176976.378800:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176976.378802:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176976.378803:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856321664, offset 0
00000400:00000200:1.0:1786176976.378806:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176976.378808:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176976.378810:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176976.378812:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176976.378814:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176976.378816:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176976.378818:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176976.378820:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254684:254684:256:4294967295] 192.168.202.110@tcp LPNI seq info [254673:254673:8:4294967295]
00000400:00000200:1.0:1786176976.378823:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176976.378827:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176976.378830:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176976.378832:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176976.378835:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176976.378837:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786176976.378840:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176976.378842:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1786176976.378845:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176976.378847:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176976.378849:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176976.378852:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176976.378853:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176976.378855:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176976.378857:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176976.378859:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378865:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176976.378867:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378871:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.378873:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176976.378875:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176976.378877:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176976.378879:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176976.378880:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301872147
00000100:00000001:1.0:1786176976.378882:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301875917
00000100:00000001:1.0:1786176976.378884:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301872147
00000100:00000001:1.0:1786176976.378885:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301875917
00000100:00000001:1.0:1786176976.378887:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176976.378888:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.378889:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176976.378891:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176976/real 1786176976]  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176976.378916:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176976.378917:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176976.378919:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176976.378922:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176976.378924:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176976.378926:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176976.378930:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176976.378940:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378954:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176976.378956:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378958:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176976.378959:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176976.378964:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378971:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176976.378974:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.378980:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176976.378981:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378983:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176976.378983:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.378985:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176976.378986:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176976.378987:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176976.378989:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176976.378991:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176976.378993:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176976.378999:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856321664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176976.379001:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.379003:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176976.379005:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b800 x1872934856321536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.379008:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176976.379010:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176976.379011:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176976.379013:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176976.379014:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176976.379016:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176976.379017:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176976.379019:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:1.0:1786176976.379020:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176976.379021:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176976.379022:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b800.
00000100:00000001:1.0:1786176976.379024:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176976.379026:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.379027:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176976.379029:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418380 x1872934856321664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176976.379033:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176976.379033:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176976.379034:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176976.379035:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176976.379036:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176976.379037:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176976.379038:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176976.379039:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:1.0:1786176976.379040:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176976.379040:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176976.379041:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418380.
00000100:00000001:1.0:1786176976.379042:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176976.379044:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.379045:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176976.379048:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176976.379049:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176976.379050:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176976.379051:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176976.952620:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176976.952625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176976.952629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176976.952630:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176976.952633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176976.952635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176976.952698:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176976.952703:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176976.952707:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176977.400596:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176977.400602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176977.400604:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176977.400608:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176977.400610:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176977.400613:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176977.400615:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176977.400617:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176977.400619:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176977.976602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176977.976607:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176977.976611:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176977.976612:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176977.976615:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176977.976616:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176977.976664:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176977.976669:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176977.976671:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176978.424615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176978.424622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176978.424624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176978.424630:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176978.424634:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176978.424637:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176978.424641:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176978.424643:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176978.424647:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176979.000627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176979.000633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176979.000637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176979.000638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176979.000641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176979.000642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176979.000704:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176979.000707:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176979.000710:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176979.448623:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176979.448630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176979.448632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176979.448636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176979.448639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176979.448642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176979.448644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176979.448647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176979.448648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176980.024637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176980.024642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176980.024645:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176980.024646:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176980.024649:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176980.024650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176980.024704:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176980.024707:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176980.024710:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786176980.216595:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301875917, at 4301875968
00000100:00000040:0.0:1786176980.216597:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301875917, at 4301875968
00000100:00000010:0.0:1786176980.216611:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1600.
00000100:00000010:1.0:1786176980.216613:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1900.
00000100:00000001:1.0:1786176980.472628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176980.472636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176980.472639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176980.472645:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176980.472649:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176980.472653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176980.472656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176980.472659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176980.472662:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176981.048639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176981.048648:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176981.048651:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176981.048654:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176981.048659:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176981.048662:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176981.048664:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176981.048667:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176981.048671:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176981.048673:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176981.048674:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176981.048684:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176981.048686:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176981.048688:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176981.048690:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176981.048692:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176981.048696:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176981.048698:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176981.048752:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176981.048757:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176981.048759:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176981.496604:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176981.496610:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176981.496612:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176981.496618:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176981.496621:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176981.496624:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.496627:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1786176981.496627:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176981.496630:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.496634:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176981.496637:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176981.496641:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176981.496646:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176981.496651:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176981.496659:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7204
00000400:00000200:2.0:1786176981.496669:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176981.496676:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176981.496680:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176981.496684:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176981.496688:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176981.496691:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176981.496693:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176981.496697:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176981.496700:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176981.496703:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176981.496707:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176981.496709:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176981.496712:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176981.496716:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176981.496718:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176981.496720:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176981.496722:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176981.496726:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176981.496729:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176981.496732:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176981.496735:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176981.496739:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176981.496742:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176981.496745:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176981.496747:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176981.496756:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000020:00000040:2.0:1786176981.496760:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176981.496764:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176981.496766:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176981.496770:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176981.496773:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176981.496777:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176981.496785:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786176981.496792:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176981.496802:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176981.496820:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176981.496825:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176981.496831:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176981.496876:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176981.496880:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176981.496883:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176981.496886:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176981.496888:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176981.496892:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176981.496895:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7204
00000100:00000001:1.0:1786176981.496901:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:2.0:1786176981.496901:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176981.496906:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000400:00000200:2.0:1786176981.496908:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176981.496909:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1786176981.496913:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786176981.496917:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.496930:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1786176981.496930:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176981.496934:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176981.496936:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1786176981.496940:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1786176981.496941:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176981.496945:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786176981.496946:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:1.0:1786176981.496952:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176981.496953:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176981.496956:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176981.496961:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00100000:1.0:1786176981.496962:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856321792:192.168.202.110@tcp:250:kworker.0
00000020:00000040:2.0:1786176981.496965:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:1.0:1786176981.496970:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:2.0:1786176981.496971:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786176981.496975:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:1.0:1786176981.496976:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:2.0:1786176981.496981:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176981.496982:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176981.496988:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1786176981.496989:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:2.0:1786176981.496994:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000010:1.0:1786176981.496997:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
00000100:00000001:2.0:1786176981.496999:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:1.0:1786176981.497000:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786176981.497007:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000400:00000200:1.0:1786176981.497011:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786176981.497015:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:1.0:1786176981.497016:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000100:00000001:2.0:1786176981.497022:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176981.497025:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
02000000:00000001:2.0:1786176981.497026:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:1.0:1786176981.497033:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856321792, portal 25
02000000:00000001:2.0:1786176981.497034:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176981.497037:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
02000000:00000001:2.0:1786176981.497039:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.497040:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454188800 : -114548255362816 : ffff97d1a7edf100)
00000100:00000001:2.0:1786176981.497046:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:1.0:1786176981.497050:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1786176981.497052:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176981.497061:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176981.497067:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.497074:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000010:2.0:1786176981.497076:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000100:00000040:1.0:1786176981.497077:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176981.497083:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000020:00000040:2.0:1786176981.497083:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000200:1.0:1786176981.497088:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856321792, offset 0
02000000:00000001:2.0:1786176981.497090:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176981.497093:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176981.497096:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176981.497099:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176981.497103:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1786176981.497105:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786176981.497105:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176981.497113:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000010:2.0:1786176981.497114:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000400:00000200:1.0:1786176981.497118:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786176981.497121:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176981.497124:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176981.497129:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176981.497133:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00100000:2.0:1786176981.497134:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176981.497137:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254685:254685:256:4294967295] 192.168.202.110@tcp LPNI seq info [254674:254674:8:4294967295]
00000400:00000200:1.0:1786176981.497147:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176981.497155:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786176981.497159:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786176981.497161:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176981.497167:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786176981.497171:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:1.0:1786176981.497174:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176981.497184:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176981.497189:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1786176981.497192:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786176981.497197:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786176981.497199:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.1:1786176981.497201:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786176981.497209:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176981.497210:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000800:00000010:1.0:1786176981.497212:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000100:00000001:2.0:1786176981.497214:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176981.497218:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786176981.497219:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1786176981.497225:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786176981.497226:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176981.497232:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000100:00080000:2.0:1786176981.497233:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7204
00000400:00000010:1.0:1786176981.497235:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176981.497240:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:2.0:1786176981.497244:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:1.0:1786176981.497246:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786176981.497254:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176981.497257:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:2.0:1786176981.497261:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786176981.497262:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176981.497270:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176981.497277:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176981.497284:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176981.497284:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176981.497288:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786176981.497290:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176981.497293:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176981.497294:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176981.497298:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000200:1.0:1786176981.497299:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1786176981.497301:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176981.497303:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301881037
00000020:00000040:2.0:1786176981.497305:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000010:1.0:1786176981.497308:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1900.
00000020:00000001:2.0:1786176981.497310:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786176981.497315:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:1.0:1786176981.497316:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:2.0:1786176981.497321:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176981.497323:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301881037
00000100:00000001:2.0:1786176981.497326:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:1.0:1786176981.497327:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1e40.
00000100:00000001:1.0:1786176981.497332:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:2.0:1786176981.497333:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176981.497334:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176981.497340:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176981.497340:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176981.497342:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000020:00000040:2.0:1786176981.497347:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00100000:1.0:1786176981.497349:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176981/real 1786176981]  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176981.497355:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000001:1.0:1786176981.497361:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786176981.497362:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176981.497364:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:2.0:1786176981.497368:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:1.0:1786176981.497369:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176981.497373:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176981.497374:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
02000000:00000001:2.0:1786176981.497375:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176981.497377:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176981.497382:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176981.497383:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176981.497389:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176981.497394:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000200:1.0:1786176981.497395:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1786176981.497402:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.497406:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786176981.497407:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.497408:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.497413:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176981.497414:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:2.0:1786176981.497416:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000100:00100000:1.0:1786176981.497418:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176981.497423:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786176981.497426:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786176981.497430:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:1.0:1786176981.497432:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786176981.497435:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176981.497442:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786176981.497443:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176981.497447:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786176981.497447:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176981.497451:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176981.497452:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786176981.497452:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.497456:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176981.497460:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176981.497462:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000010:2.0:1786176981.497463:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00080000:1.0:1786176981.497465:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176981.497469:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1786176981.497471:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176981.497472:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176981.497483:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856321792:192.168.202.110@tcp:250:kworker.0
00000100:00100000:2.0:1786176981.497485:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176981.497487:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.497492:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176981.497496:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf100 x1872934856321792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176981.497507:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176981.497529:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176981.497532:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176981.497534:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176981.497537:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176981.497538:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176981.497540:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176981.497542:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176981.497545:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:1.0:1786176981.497549:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000001:2.0:1786176981.497551:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176981.497552:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176981.497554:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf100.
00000100:00000001:1.0:1786176981.497558:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176981.497560:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.497562:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786176981.497563:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.497568:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176981.497571:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.497574:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176981.497577:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176981.497579:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176981.497585:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176981.497585:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176981.497593:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176981.497594:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176981.497597:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176981.497600:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176981.497603:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:2.0:1786176981.497603:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
02000000:00000001:1.0:1786176981.497610:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176981.497610:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:1.0:1786176981.497614:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856321920:192.168.202.110@tcp:38:kworker.0
00000100:00080000:2.0:1786176981.497615:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176981.497620:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176981.497623:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1786176981.497623:0:62275:0:(import.c:485:import_select_connection()) Process entered
02000000:00000001:1.0:1786176981.497625:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176981.497627:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176981.497630:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
00000100:00080000:2.0:1786176981.497631:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7204
02000000:00000001:1.0:1786176981.497633:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176981.497641:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176981.497643:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176981.497644:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176981.497650:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000400:00000200:2.0:1786176981.497654:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000200:1.0:1786176981.497655:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856321920, portal 10
00000100:00000001:1.0:1786176981.497660:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1786176981.497661:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176981.497662:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000100:00000040:2.0:1786176981.497669:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176981.497671:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176981.497674:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176981.497679:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176981.497682:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176981.497685:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176981.497687:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:1.0:1786176981.497691:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:2.0:1786176981.497691:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000040:1.0:1786176981.497696:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:2.0:1786176981.497697:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000400:00000010:1.0:1786176981.497699:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000020:00000001:2.0:1786176981.497704:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000200:1.0:1786176981.497706:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856321920, offset 0
00000020:00000001:2.0:1786176981.497711:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:1.0:1786176981.497712:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176981.497718:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786176981.497722:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176981.497724:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1786176981.497728:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176981.497729:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176981.497736:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:2.0:1786176981.497737:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176981.497739:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176981.497744:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1786176981.497744:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176981.497750:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254686:254686:256:4294967295] 192.168.202.110@tcp LPNI seq info [254675:254675:8:4294967295]
00000020:00000040:2.0:1786176981.497752:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000400:00000200:1.0:1786176981.497755:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786176981.497761:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1786176981.497764:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786176981.497770:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176981.497772:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176981.497777:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
02000000:00000001:2.0:1786176981.497777:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:1.0:1786176981.497782:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1786176981.497783:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:1.0:1786176981.497785:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:2.0:1786176981.497791:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786176981.497792:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176981.497796:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:2.0:1786176981.497798:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000800:00000100:1.0:1786176981.497802:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176981.497805:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000008:00100000:2.0:1786176981.497805:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:1.0:1786176981.497809:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00000001:2.0:1786176981.497812:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176981.497814:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1786176981.497818:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176981.497819:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176981.497824:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176981.497826:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000010:2.0:1786176981.497827:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000100:00000200:1.0:1786176981.497830:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176981.497833:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176981.497840:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176981.497841:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1786176981.497862:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176981.497863:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176981.497869:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786176981.497874:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176981.497874:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176981.497878:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1786176981.497880:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.497883:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176981.497888:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
02000000:00000010:2.0:1786176981.497888:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000400:00000200:1.0:1786176981.497893:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176981.497895:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301877258
00000100:00000001:2.0:1786176981.497896:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.497898:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301881037
00000100:00000001:1.0:1786176981.497902:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301877258
00000100:00000001:1.0:1786176981.497904:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301881037
00000100:00000001:1.0:1786176981.497907:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1786176981.497908:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176981.497909:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.497912:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176981.497918:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176981/real 1786176981]  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176981.497932:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000001:1.0:1786176981.497937:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786176981.497938:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176981.497940:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176981.497944:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000100:00000040:2.0:1786176981.497947:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176981.497949:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176981.497953:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176981.497958:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176981.497966:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176981.497967:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176981.497974:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176981.497987:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000200:1.0:1786176981.497990:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176981.497991:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176981.497995:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786176981.497998:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176981.498001:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176981.498001:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176981.498006:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176981.498008:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786176981.498010:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7204
00000100:00100000:1.0:1786176981.498013:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786176981.498022:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176981.498024:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1786176981.498033:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:1.0:1786176981.498034:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176981.498040:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176981.498045:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:2.0:1786176981.498049:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176981.498050:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498053:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176981.498056:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176981.498056:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176981.498060:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000040:2.0:1786176981.498063:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176981.498064:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786176981.498069:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00080000:1.0:1786176981.498070:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000020:00000001:2.0:1786176981.498075:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00080000:1.0:1786176981.498076:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176981.498080:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000020:00000040:2.0:1786176981.498080:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176981.498084:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000040:1.0:1786176981.498086:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786176981.498089:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786176981.498096:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00100000:1.0:1786176981.498098:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856321920:192.168.202.110@tcp:38:kworker.0
00000020:00000001:2.0:1786176981.498101:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176981.498103:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498108:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786176981.498111:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176981.498113:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856321920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176981.498116:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176981.498122:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000040:2.0:1786176981.498123:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176981.498124:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176981.498128:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
00000100:00000001:2.0:1786176981.498129:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:1.0:1786176981.498131:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176981.498134:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786176981.498134:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000020:00000040:1.0:1786176981.498136:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176981.498139:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00080000:2.0:1786176981.498141:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000010:1.0:1786176981.498142:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:1.0:1786176981.498148:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000001:2.0:1786176981.498149:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176981.498150:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176981.498153:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
02000000:00000001:2.0:1786176981.498154:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176981.498156:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176981.498158:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786176981.498160:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176981.498161:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498166:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:2.0:1786176981.498166:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.498168:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176981.498171:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176981.498172:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176981.498174:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176981.498176:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000008:00100000:2.0:1786176981.498177:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000040:1.0:1786176981.498180:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1786176981.498184:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176981.498191:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.498192:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176981.498195:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176981.498197:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176981.498199:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:2.0:1786176981.498199:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
02000000:00000001:1.0:1786176981.498203:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786176981.498206:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00100000:1.0:1786176981.498207:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322048:192.168.202.110@tcp:38:kworker.0
02000000:00000001:2.0:1786176981.498212:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176981.498213:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176981.498216:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786176981.498217:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176981.498218:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176981.498222:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786176981.498222:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176981.498227:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:1.0:1786176981.498228:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:2.0:1786176981.498230:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176981.498231:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176981.498238:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:2.0:1786176981.498240:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000400:00000010:1.0:1786176981.498241:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176981.498247:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000001:2.0:1786176981.498248:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176981.498250:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856322048, portal 10
00000100:00000001:1.0:1786176981.498253:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176981.498254:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00100000:2.0:1786176981.498259:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1786176981.498263:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498288:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786176981.498289:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1786176981.498292:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786176981.498294:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176981.498295:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176981.498300:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856322048, offset 0
00000100:00000040:2.0:1786176981.498304:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176981.498305:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176981.498312:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176981.498318:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176981.498325:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176981.498329:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1786176981.498330:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176981.498333:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176981.498336:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:2.0:1786176981.498337:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7214)
00000400:00000200:1.0:1786176981.498339:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254687:254687:256:4294967295] 192.168.202.110@tcp LPNI seq info [254676:254676:8:4294967295]
00000400:00000200:1.0:1786176981.498344:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176981.498350:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176981.498353:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176981.498355:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176981.498358:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176981.498360:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786176981.498364:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176981.498365:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786176981.498368:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176981.498369:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176981.498371:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176981.498374:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176981.498376:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176981.498377:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176981.498379:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176981.498381:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498387:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176981.498389:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498393:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.498394:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176981.498396:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176981.498398:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176981.498401:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176981.498402:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301877383
00000100:00000001:1.0:1786176981.498404:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301881037
00000100:00000001:1.0:1786176981.498406:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301877383
00000100:00000001:1.0:1786176981.498407:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301881037
00000100:00000001:1.0:1786176981.498409:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176981.498410:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.498411:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176981.498413:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176981/real 1786176981]  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176981.498418:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176981.498419:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176981.498420:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176981.498421:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176981.498422:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176981.498424:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176981.498426:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176981.498431:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498435:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176981.498436:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498437:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176981.498437:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176981.498440:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498444:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176981.498446:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498451:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176981.498452:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498453:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176981.498454:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498456:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176981.498457:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176981.498459:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176981.498461:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176981.498462:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176981.498464:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176981.498469:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176981.498471:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176981.498473:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498478:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176981.498479:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176981.498481:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176981.498482:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176981.498483:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176981.498484:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176981.498486:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176981.498487:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176981.498488:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176981.498489:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176981.498491:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176981.498493:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176981.498495:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176981.498496:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176981.498498:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176981.498500:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856322176, portal 4
00000100:00000001:1.0:1786176981.498501:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176981.498503:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178944 : -114548255372672 : ffff97d1a7edca80)
00000100:00000040:1.0:1786176981.498506:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498526:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176981.498528:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176981.498530:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176981.498532:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856322176, offset 0
00000400:00000200:1.0:1786176981.498534:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176981.498537:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176981.498539:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176981.498541:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176981.498543:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176981.498545:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176981.498546:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176981.498548:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254688:254688:256:4294967295] 192.168.202.110@tcp LPNI seq info [254677:254677:8:4294967295]
00000400:00000200:1.0:1786176981.498551:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176981.498555:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176981.498558:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176981.498560:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176981.498563:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176981.498565:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786176981.498567:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176981.498569:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786176981.498572:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176981.498573:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176981.498575:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176981.498577:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176981.498579:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176981.498580:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176981.498582:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176981.498584:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498589:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176981.498592:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498596:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.498597:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176981.498599:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176981.498601:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176981.498603:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176981.498604:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301877259
00000100:00000001:1.0:1786176981.498606:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301881037
00000100:00000001:1.0:1786176981.498607:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301877259
00000100:00000001:1.0:1786176981.498609:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301881037
00000100:00000001:1.0:1786176981.498610:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176981.498611:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.498612:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176981.498614:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176981/real 1786176981]  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176981.498619:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176981.498620:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176981.498621:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176981.498622:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176981.498623:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176981.498625:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176981.498627:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176981.498633:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498636:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176981.498637:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498638:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176981.498639:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176981.498641:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498646:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176981.498648:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498652:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176981.498654:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498655:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176981.498655:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498656:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176981.498657:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176981.498659:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176981.498661:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176981.498662:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176981.498664:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176981.498669:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176981.498671:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498672:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176981.498674:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856322048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498678:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176981.498679:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176981.498680:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176981.498682:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176981.498683:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176981.498684:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176981.498685:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176981.498687:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:1.0:1786176981.498688:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176981.498689:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176981.498690:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:1.0:1786176981.498692:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176981.498693:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498694:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176981.498696:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edca80 x1872934856322176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498700:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176981.498701:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176981.498702:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176981.498703:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176981.498704:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176981.498704:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176981.498705:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176981.498706:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:1.0:1786176981.498708:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176981.498708:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176981.498709:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edca80.
00000100:00000001:1.0:1786176981.498710:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176981.498711:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498712:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498714:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176981.498715:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.498716:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176981.498717:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176981.498718:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176981.498720:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498724:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176981.498725:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176981.498726:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176981.498728:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176981.498729:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176981.498731:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176981.498733:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176981.498734:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176981.498735:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176981.498736:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176981.498738:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176981.498739:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176981.498741:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176981.498742:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176981.498744:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176981.498746:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856322304, portal 4
00000100:00000001:1.0:1786176981.498748:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176981.498749:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000100:00000040:1.0:1786176981.498752:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498757:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176981.498758:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176981.498759:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176981.498761:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856322304, offset 0
00000400:00000200:1.0:1786176981.498763:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176981.498765:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176981.498767:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176981.498769:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176981.498770:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176981.498772:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176981.498774:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176981.498776:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254689:254689:256:4294967295] 192.168.202.110@tcp LPNI seq info [254678:254678:8:4294967295]
00000400:00000200:1.0:1786176981.498778:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176981.498782:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176981.498785:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176981.498787:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176981.498789:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176981.498791:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786176981.498794:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176981.498795:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786176981.498798:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176981.498799:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176981.498801:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176981.498803:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176981.498805:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176981.498806:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176981.498808:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176981.498810:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498814:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176981.498816:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177006 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498820:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.498821:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176981.498823:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176981.498825:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176981.498826:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176981.498827:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301877267
00000100:00000001:1.0:1786176981.498829:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301881037
00000100:00000001:1.0:1786176981.498830:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301877267
00000100:00000001:1.0:1786176981.498832:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301881037
00000100:00000001:1.0:1786176981.498833:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176981.498834:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.498836:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176981.498838:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176981/real 1786176981]  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176981.498842:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176981.498843:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176981.498844:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176981.498862:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176981.498863:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176981.498865:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176981.498867:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176981.498873:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498877:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176981.498878:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498880:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176981.498880:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176981.498883:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498887:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176981.498890:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498894:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176981.498895:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498896:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176981.498897:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498898:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176981.498899:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176981.498901:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176981.498903:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176981.498904:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176981.498906:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176981.498911:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176981.498914:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498915:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176981.498917:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856322304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177006 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176981.498922:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176981.498923:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176981.498924:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176981.498927:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176981.498928:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176981.498929:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176981.498931:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176981.498933:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:1.0:1786176981.498935:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176981.498936:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176981.498938:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
00000100:00000001:1.0:1786176981.498940:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176981.498941:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498943:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176981.498947:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176981.498948:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176981.498950:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176981.498952:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176982.072641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176982.072648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176982.072653:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176982.072654:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176982.072656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176982.072657:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176982.072708:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176982.072711:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176982.072714:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176982.520634:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176982.520643:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176982.520646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176982.520651:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176982.520655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176982.520660:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176982.520663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176982.520666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176982.520671:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176983.096597:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176983.096602:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176983.096606:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176983.096607:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176983.096610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176983.096612:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176983.096614:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176983.096616:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176983.096619:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176983.096620:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176983.096622:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176983.096623:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176983.096687:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176983.096691:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176983.096694:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176983.544640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176983.544648:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176983.544650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176983.544656:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176983.544660:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176983.544664:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176983.544668:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176983.544670:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176983.544674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176984.120645:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176984.120650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176984.120654:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176984.120655:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176984.120657:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176984.120658:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176984.120736:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176984.120741:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176984.120745:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1786176984.568582:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786176984.568587:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176984.568589:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176984.568593:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786176984.568596:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176984.568599:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176984.568601:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176984.568603:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176984.568605:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176985.144629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176985.144634:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176985.144638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176985.144639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176985.144641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176985.144642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176985.144706:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176985.144709:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176985.144712:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176985.336624:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301881037, at 4301881088
00000100:00000040:1.0:1786176985.336624:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301881037, at 4301881088
00000100:00000010:0.0:1786176985.336643:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1900.
00000100:00000010:1.0:1786176985.336645:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1e40.
00000008:00000020:1.0:1786176985.592683:0:104859:0:(osc_request.c:745:osc_update_next_shrink()) next time 8414 to shrink grant
00000100:00000001:3.0:1786176985.592688:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000008:00000020:1.0:1786176985.592694:0:104859:0:(osc_request.c:745:osc_update_next_shrink()) next time 8414 to shrink grant
00000100:00000001:3.0:1786176985.592696:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176985.592700:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176985.592704:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176985.592716:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176985.592720:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176985.592724:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176985.592727:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176985.592730:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176986.168639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176986.168646:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176986.168651:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176986.168652:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176986.168654:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176986.168655:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176986.168756:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176986.168760:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176986.168764:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786176986.616634:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786176986.616650:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1786176986.616650:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786176986.616658:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1786176986.616659:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176986.616663:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176986.616668:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1786176986.616669:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176986.616677:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176986.616681:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786176986.616682:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176986.616687:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176986.616690:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176986.616693:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176986.616696:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7209
00000400:00000200:3.0:1786176986.616710:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176986.616728:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176986.616733:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176986.616737:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.616742:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176986.616745:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.616747:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176986.616753:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176986.616755:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176986.616759:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176986.616763:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176986.616766:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176986.616768:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176986.616773:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.616776:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176986.616779:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.616781:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176986.616784:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176986.616788:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176986.616791:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176986.616794:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176986.616811:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176986.616815:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.616818:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176986.616820:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176986.616832:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f800.
00000020:00000040:3.0:1786176986.616837:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176986.616842:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176986.616845:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.616849:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176986.616853:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176986.616859:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176986.616878:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786176986.616888:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176986.616900:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176986.616912:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176986.616925:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176986.616931:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4d339d9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176986.616956:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176986.616960:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176986.616964:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176986.616966:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176986.616969:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176986.616972:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176986.616975:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7209
00000400:00000200:3.0:1786176986.616980:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176986.616984:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176986.616987:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176986.616991:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.616995:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176986.616997:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.617000:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176986.617001:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1786176986.617004:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176986.617006:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1786176986.617008:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:1.0:1786176986.617010:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1786176986.617012:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176986.617018:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000040:1.0:1786176986.617019:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786176986.617023:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176986.617031:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176986.617036:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176986.617039:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176986.617040:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176986.617045:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1786176986.617046:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1786176986.617049:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1786176986.617052:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176986.617053:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.617059:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00100000:1.0:1786176986.617062:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856322432:192.168.202.110@tcp:250:kworker.0
00000020:00000040:3.0:1786176986.617064:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786176986.617068:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:3.0:1786176986.617072:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:1.0:1786176986.617074:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176986.617077:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.617078:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176986.617081:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1786176986.617083:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:1.0:1786176986.617087:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176986.617090:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176986.617092:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786176986.617098:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786176986.617100:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176986.617103:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000100:00000001:3.0:1786176986.617105:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000010:1.0:1786176986.617112:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000008:00100000:3.0:1786176986.617112:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176986.617119:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.617122:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176986.617125:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856322432, portal 25
00000100:00000010:3.0:1786176986.617128:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69fb80.
00000100:00000001:1.0:1786176986.617130:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000020:00000040:3.0:1786176986.617133:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786176986.617135:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
02000000:00000001:3.0:1786176986.617138:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176986.617142:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176986.617146:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176986.617147:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176986.617151:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176986.617154:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176986.617163:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:1.0:1786176986.617164:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786176986.617169:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176986.617171:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176986.617178:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00100000:3.0:1786176986.617182:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1786176986.617183:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856322432, offset 0
00000400:00000200:1.0:1786176986.617196:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176986.617197:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176986.617201:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176986.617207:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1786176986.617208:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014f55bbb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786176986.617217:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1786176986.617226:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176986.617227:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1786176986.617233:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786176986.617235:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:3.0:1786176986.617241:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:1.0:1786176986.617242:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1786176986.617245:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1786176986.617247:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:3.0:1786176986.617249:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786176986.617255:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254690:254690:256:4294967295] 192.168.202.110@tcp LPNI seq info [254679:254679:8:4294967295]
00000100:00000001:3.0:1786176986.617257:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786176986.617265:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1786176986.617265:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7209
00000400:00000200:3.0:1786176986.617274:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176986.617281:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176986.617286:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786176986.617290:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1786176986.617295:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.617300:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1786176986.617304:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786176986.617306:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.617311:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176986.617314:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000010:1.0:1786176986.617315:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000020:00000040:3.0:1786176986.617317:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786176986.617322:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176986.617325:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786176986.617328:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786176986.617331:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:1.0:1786176986.617334:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1786176986.617336:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.617341:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1786176986.617343:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000100:00000040:3.0:1786176986.617345:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.617350:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176986.617355:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000800:00000200:1.1:1786176986.617357:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786176986.617362:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:1.1:1786176986.617366:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000100:00000001:3.0:1786176986.617368:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176986.617374:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176986.617377:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000100:1.0:1786176986.617378:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786176986.617384:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786176986.617385:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000100:00000001:3.0:1786176986.617387:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176986.617391:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176986.617395:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.617397:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176986.617400:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ca80.
00000020:00000040:3.0:1786176986.617403:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176986.617405:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:1.0:1786176986.617406:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:3.0:1786176986.617408:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.617414:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176986.617416:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1786176986.617421:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:3.0:1786176986.617421:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176986.617430:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000400:00000200:1.0:1786176986.617433:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000100:00000001:3.0:1786176986.617436:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176986.617438:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00100000:3.0:1786176986.617446:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176986.617447:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176986.617455:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176986.617481:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176986.617483:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786176986.617486:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176986.617491:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176986.617496:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ffb0865] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.617541:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.617545:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.617546:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:3.0:1786176986.617550:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176986.617556:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1786176986.617558:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.617559:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176986.617564:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786176986.617569:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176986.617571:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176986.617575:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301886158
00000100:00080000:3.0:1786176986.617578:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7209
00000100:00000010:1.0:1786176986.617585:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1e40.
00000400:00000200:3.0:1786176986.617586:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176986.617593:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176986.617597:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:3.0:1786176986.617597:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176986.617603:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.617607:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176986.617610:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.617612:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176986.617615:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176986.617616:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301886158
00000020:00000040:3.0:1786176986.617617:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786176986.617623:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000010:1.0:1786176986.617624:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1420.
00000020:00000040:3.0:1786176986.617626:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:1.0:1786176986.617631:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000020:00000001:3.0:1786176986.617632:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786176986.617637:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786176986.617640:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1786176986.617643:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176986.617646:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.617647:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176986.617651:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176986.617654:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1786176986.617655:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176986.617659:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00100000:1.0:1786176986.617664:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176986/real 1786176986]  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176986.617665:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176986.617670:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176986.617676:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176986.617681:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176986.617684:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.617687:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176986.617690:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000001:1.0:1786176986.617692:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000008:00000001:3.0:1786176986.617694:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176986.617696:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1786176986.617698:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176986.617701:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000100:00000010:3.0:1786176986.617705:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f480.
00000400:00000200:1.0:1786176986.617707:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000020:00000040:3.0:1786176986.617709:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000400:00000010:1.0:1786176986.617711:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
02000000:00000001:3.0:1786176986.617715:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176986.617718:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:3.0:1786176986.617718:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.617723:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176986.617725:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:1.0:1786176986.617727:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176986.617728:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176986.617735:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:3.0:1786176986.617742:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176986.617746:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176986.617752:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786176986.617765:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786176986.617769:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786176986.617771:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176986.617774:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.617776:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176986.617781:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176986.617782:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092aa1aa7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176986.617790:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176986.617794:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176986.617815:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176986.617819:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786176986.617820:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786176986.617822:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176986.617825:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176986.617828:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:1.0:1786176986.617830:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786176986.617832:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7209
00000400:00000200:3.0:1786176986.617841:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176986.617846:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176986.617850:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786176986.617854:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:3.0:1786176986.617855:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.617859:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176986.617862:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176986.617863:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176986.617868:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786176986.617869:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176986.617873:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:3.0:1786176986.617873:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176986.617876:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:1.0:1786176986.617878:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000001:3.0:1786176986.617878:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176986.617883:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:1.0:1786176986.617884:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:3.0:1786176986.617885:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786176986.617891:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00080000:1.0:1786176986.617892:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:3.0:1786176986.617894:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176986.617899:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176986.617903:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00080000:1.0:1786176986.617904:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786176986.617905:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176986.617910:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:1.0:1786176986.617911:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00080000:3.0:1786176986.617913:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176986.617918:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176986.617920:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1786176986.617923:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176986.617924:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176986.617928:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.617933:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176986.617935:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176986.617940:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176986.617943:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176986.617948:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f100.
00000100:00100000:1.0:1786176986.617950:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856322432:192.168.202.110@tcp:250:kworker.0
00000020:00000040:3.0:1786176986.617952:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176986.617957:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176986.617959:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.617962:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176986.617962:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176986.617964:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176986.617968:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1786176986.617968:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176986.617975:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000040:1.0:1786176986.617976:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856322432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176986.617983:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.617992:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00100000:3.0:1786176986.617995:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786176986.617996:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176986.618001:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176986.618010:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:3.0:1786176986.618010:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000020:00000001:1.0:1786176986.618015:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000001:3.0:1786176986.618015:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1786176986.618021:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000040:3.0:1786176986.618022:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1ef9f45] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1786176986.618025:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176986.618030:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:1.0:1786176986.618037:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000001:3.0:1786176986.618039:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176986.618042:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176986.618045:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000040:3.0:1786176986.618048:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7220)
00000100:00000001:1.0:1786176986.618050:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176986.618053:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.618057:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.618067:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176986.618069:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.618072:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176986.618074:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176986.618075:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176986.618080:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618088:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176986.618089:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176986.618092:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176986.618094:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176986.618097:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176986.618101:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176986.618104:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176986.618106:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176986.618108:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176986.618110:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176986.618114:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176986.618116:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176986.618120:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176986.618124:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176986.618128:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176986.618132:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856322560, portal 10
00000100:00000001:1.0:1786176986.618134:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176986.618136:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000040:1.0:1786176986.618143:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618151:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176986.618154:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176986.618156:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176986.618159:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856322560, offset 0
00000400:00000200:1.0:1786176986.618163:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176986.618168:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176986.618172:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176986.618176:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176986.618179:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176986.618181:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176986.618184:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176986.618187:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254691:254691:256:4294967295] 192.168.202.110@tcp LPNI seq info [254680:254680:8:4294967295]
00000400:00000200:1.0:1786176986.618192:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176986.618199:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176986.618204:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176986.618208:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176986.618213:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176986.618216:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1786176986.618222:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176986.618225:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:1.0:1786176986.618229:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176986.618231:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176986.618235:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176986.618239:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176986.618243:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176986.618245:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176986.618248:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176986.618252:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618260:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176986.618264:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618279:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.618281:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176986.618285:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176986.618289:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176986.618293:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176986.618295:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301882378
00000100:00000001:1.0:1786176986.618298:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301886158
00000100:00000001:1.0:1786176986.618301:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301882378
00000100:00000001:1.0:1786176986.618303:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301886158
00000100:00000001:1.0:1786176986.618305:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176986.618307:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.618309:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176986.618313:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176986/real 1786176986]  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176986.618322:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176986.618324:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176986.618326:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176986.618328:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176986.618330:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176986.618332:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176986.618336:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176986.618345:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618351:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176986.618353:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.618355:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176986.618356:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176986.618360:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618367:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176986.618371:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618378:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176986.618381:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.618382:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176986.618384:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.618385:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176986.618387:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176986.618390:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176986.618393:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176986.618396:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176986.618400:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176986.618409:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176986.618413:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176986.618416:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618424:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176986.618426:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176986.618428:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176986.618430:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176986.618432:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176986.618435:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176986.618439:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176986.618441:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176986.618443:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176986.618445:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176986.618449:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176986.618451:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176986.618454:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176986.618457:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176986.618460:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176986.618464:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856322688, portal 10
00000100:00000001:1.0:1786176986.618466:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176986.618468:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:1.0:1786176986.618474:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618481:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176986.618483:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176986.618486:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176986.618488:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856322688, offset 0
00000400:00000200:1.0:1786176986.618492:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176986.618496:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176986.618500:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176986.618503:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176986.618506:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176986.618508:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176986.618531:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176986.618535:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254692:254692:256:4294967295] 192.168.202.110@tcp LPNI seq info [254681:254681:8:4294967295]
00000400:00000200:1.0:1786176986.618540:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176986.618546:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176986.618552:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176986.618556:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176986.618560:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176986.618563:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1786176986.618567:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176986.618570:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:1.0:1786176986.618576:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176986.618578:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176986.618582:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176986.618586:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176986.618589:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176986.618591:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176986.618593:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176986.618598:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618607:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176986.618610:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618617:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.618619:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176986.618622:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176986.618625:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176986.618628:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176986.618630:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301882504
00000100:00000001:1.0:1786176986.618634:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301886158
00000100:00000001:1.0:1786176986.618636:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301882504
00000100:00000001:1.0:1786176986.618638:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301886158
00000100:00000001:1.0:1786176986.618641:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176986.618643:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.618644:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176986.618648:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176986/real 1786176986]  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176986.618657:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176986.618659:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176986.618660:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176986.618663:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176986.618665:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176986.618668:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176986.618671:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176986.618680:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618687:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176986.618689:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.618691:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176986.618693:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176986.618696:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618704:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176986.618708:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618715:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176986.618718:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.618720:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176986.618721:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.618723:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176986.618724:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176986.618727:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176986.618730:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176986.618733:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176986.618737:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176986.618746:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176986.618750:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176986.618754:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618761:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176986.618763:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176986.618765:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176986.618767:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176986.618769:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176986.618772:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176986.618776:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176986.618778:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176986.618779:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176986.618781:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176986.618785:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176986.618788:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176986.618791:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176986.618793:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176986.618813:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176986.618816:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856322816, portal 4
00000100:00000001:1.0:1786176986.618819:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176986.618820:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:1.0:1786176986.618827:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618835:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176986.618837:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176986.618840:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176986.618842:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856322816, offset 0
00000400:00000200:1.0:1786176986.618847:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176986.618851:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176986.618854:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176986.618858:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176986.618861:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176986.618863:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176986.618865:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176986.618870:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254693:254693:256:4294967295] 192.168.202.110@tcp LPNI seq info [254682:254682:8:4294967295]
00000400:00000200:1.0:1786176986.618874:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176986.618881:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176986.618886:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176986.618889:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176986.618893:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176986.618896:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1786176986.618901:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176986.618904:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:1.0:1786176986.618908:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176986.618910:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176986.618913:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176986.618917:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176986.618921:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176986.618923:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176986.618925:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176986.618929:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618936:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176986.618940:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.618947:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.618949:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176986.618952:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176986.618956:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176986.618959:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176986.618961:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301882379
00000100:00000001:1.0:1786176986.618963:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301886158
00000100:00000001:1.0:1786176986.618965:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301882379
00000100:00000001:1.0:1786176986.618968:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301886158
00000100:00000001:1.0:1786176986.618970:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176986.618973:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.618974:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176986.618978:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176986/real 1786176986]  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176986.618985:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176986.618987:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176986.618988:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176986.618991:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176986.618992:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176986.618994:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176986.618998:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176986.619005:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619011:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176986.619013:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.619014:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176986.619015:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176986.619018:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619024:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176986.619028:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619034:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176986.619036:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.619038:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176986.619039:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.619041:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176986.619042:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176986.619044:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176986.619048:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176986.619050:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176986.619054:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176986.619060:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176986.619064:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176986.619066:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619072:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176986.619074:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176986.619076:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176986.619078:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176986.619080:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176986.619083:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176986.619085:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176986.619088:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176986.619089:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176986.619091:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176986.619094:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176986.619096:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176986.619099:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176986.619101:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176986.619104:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176986.619107:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856322944, portal 4
00000100:00000001:1.0:1786176986.619109:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176986.619110:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:1.0:1786176986.619116:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619122:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176986.619124:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176986.619126:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176986.619128:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856322944, offset 0
00000400:00000200:1.0:1786176986.619132:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176986.619136:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176986.619138:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176986.619141:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176986.619144:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176986.619146:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176986.619148:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176986.619151:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254694:254694:256:4294967295] 192.168.202.110@tcp LPNI seq info [254683:254683:8:4294967295]
00000400:00000200:1.0:1786176986.619155:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176986.619161:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176986.619165:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176986.619168:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176986.619171:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176986.619174:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1786176986.619178:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176986.619181:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:1.0:1786176986.619185:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176986.619187:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176986.619190:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176986.619193:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176986.619196:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176986.619198:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176986.619200:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176986.619204:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619212:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176986.619215:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177011 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619222:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.619223:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176986.619226:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176986.619229:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176986.619232:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176986.619234:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301882387
00000100:00000001:1.0:1786176986.619237:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301886158
00000100:00000001:1.0:1786176986.619239:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301882387
00000100:00000001:1.0:1786176986.619241:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301886158
00000100:00000001:1.0:1786176986.619243:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176986.619245:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.619246:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176986.619250:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176986/real 1786176986]  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176986.619257:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176986.619258:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176986.619260:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176986.619262:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176986.619263:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176986.619265:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176986.619269:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176986.619277:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619282:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176986.619283:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.619284:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176986.619286:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176986.619289:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619297:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176986.619301:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619308:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176986.619310:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.619312:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176986.619313:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.619314:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176986.619316:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176986.619318:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176986.619321:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176986.619323:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176986.619326:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176986.619333:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856322944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176986.619336:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.619338:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176986.619341:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856322560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619348:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176986.619350:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176986.619352:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176986.619356:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176986.619357:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176986.619359:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176986.619362:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176986.619363:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:1.0:1786176986.619366:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176986.619368:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176986.619369:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:1.0:1786176986.619372:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176986.619373:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.619375:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176986.619379:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856322688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619385:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176986.619387:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176986.619388:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176986.619391:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176986.619392:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176986.619393:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176986.619396:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176986.619397:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176986.619400:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176986.619401:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176986.619402:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:1.0:1786176986.619405:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176986.619406:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.619407:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176986.619411:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856322816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619418:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176986.619419:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176986.619420:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176986.619423:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176986.619424:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176986.619425:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176986.619428:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176986.619429:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:1.0:1786176986.619431:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176986.619432:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176986.619434:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:1.0:1786176986.619436:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176986.619438:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.619439:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176986.619443:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856322944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177011 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176986.619456:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176986.619457:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176986.619458:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176986.619460:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176986.619462:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176986.619463:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176986.619466:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176986.619467:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:1.0:1786176986.619469:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176986.619471:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176986.619472:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:1.0:1786176986.619474:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176986.619475:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.619478:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176986.619481:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176986.619482:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176986.619484:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176986.619486:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176987.192692:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176987.192702:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176987.192709:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176987.192711:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176987.192714:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176987.192716:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176987.192808:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176987.192813:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176987.192818:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176987.640641:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176987.640647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176987.640649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176987.640654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176987.640657:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176987.640661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176987.640663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176987.640665:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176987.640667:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176988.216608:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176988.216615:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176988.216621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176988.216622:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176988.216624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176988.216625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176988.216694:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176988.216699:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176988.216702:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176988.664595:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176988.664604:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176988.664606:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176988.664612:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176988.664616:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176988.664620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176988.664623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176988.664626:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176988.664628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176989.240695:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176989.240706:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176989.240713:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176989.240715:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176989.240719:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176989.240721:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176989.240820:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176989.240826:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176989.240829:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176989.688618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176989.688627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176989.688629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176989.688635:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176989.688639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176989.688644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176989.688647:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176989.688650:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176989.688652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176990.264585:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176990.264591:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176990.264595:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176990.264596:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176990.264598:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176990.264599:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176990.264658:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176990.264663:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176990.264667:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786176990.456619:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301886158, at 4301886208
00000100:00000040:1.0:1786176990.456619:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301886158, at 4301886208
00000100:00000010:0.0:1786176990.456633:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1e40.
00000100:00000010:1.0:1786176990.456636:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1420.
00000800:00000200:3.0:1786176990.584625:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:3.0:1786176990.584635:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:3.0:1786176990.584641:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1786176990.584830:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786176990.584843:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1786176990.584849:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786176990.584852:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:1.0:1786176990.584859:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00000001:2.0:1786176990.712610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176990.712616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176990.712618:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176990.712623:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176990.712626:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176990.712629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176990.712631:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176990.712633:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176990.712635:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176991.288622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176991.288628:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176991.288629:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176991.288632:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176991.288636:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176991.288638:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176991.288640:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176991.288641:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176991.288644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176991.288645:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176991.288646:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176991.288647:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176991.288648:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176991.288649:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176991.288650:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176991.288652:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176991.288654:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176991.288655:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176991.288714:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176991.288718:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176991.288721:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786176991.736591:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176991.736599:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786176991.736602:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176991.736605:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176991.736609:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176991.736616:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7215
00000100:00000001:2.0:1786176991.736624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000400:00000200:3.0:1786176991.736624:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176991.736629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176991.736631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786176991.736633:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00100000:2.0:1786176991.736636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:3.0:1786176991.736637:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1786176991.736641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1786176991.736642:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176991.736645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176991.736646:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1786176991.736649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:3.0:1786176991.736650:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176991.736652:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176991.736653:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:2.0:1786176991.736656:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176991.736658:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176991.736661:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786176991.736663:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176991.736666:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786176991.736667:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786176991.736670:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786176991.736672:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.736673:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176991.736675:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.736676:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176991.736678:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786176991.736680:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176991.736682:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176991.736684:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176991.736687:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176991.736689:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176991.736690:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786176991.736692:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176991.736700:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d500.
00000020:00000040:3.0:1786176991.736702:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786176991.736705:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176991.736707:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176991.736709:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176991.736711:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176991.736715:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176991.736721:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:3.0:1786176991.736726:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176991.736733:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176991.736741:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176991.736744:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176991.736761:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7c36393] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176991.736778:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176991.736781:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786176991.736783:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786176991.736784:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176991.736785:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176991.736787:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176991.736789:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7215
00000400:00000200:3.0:1786176991.736791:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176991.736793:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176991.736795:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176991.736797:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.736798:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176991.736800:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.736802:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176991.736804:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176991.736805:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786176991.736807:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176991.736808:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786176991.736810:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786176991.736811:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786176991.736815:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176991.736818:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786176991.736818:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176991.736820:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176991.736825:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176991.736825:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176991.736830:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1786176991.736831:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786176991.736837:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176991.736841:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176991.736843:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176991.736845:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176991.736851:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176991.736856:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.736859:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176991.736860:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786176991.736862:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1786176991.736865:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786176991.736866:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176991.736871:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:3.0:1786176991.736871:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176991.736876:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176991.736877:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176991.736881:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ea00.
00000100:00100000:1.0:1786176991.736882:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856323072:192.168.202.110@tcp:250:kworker.0
00000020:00000040:3.0:1786176991.736886:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786176991.736888:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786176991.736890:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786176991.736892:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176991.736893:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1786176991.736894:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786176991.736897:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176991.736897:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176991.736899:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176991.736901:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786176991.736904:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000010:1.0:1786176991.736908:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
00000100:00000001:3.0:1786176991.736908:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176991.736914:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176991.736918:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176991.736923:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176991.736930:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000100:00000001:3.0:1786176991.736931:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176991.736934:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176991.736937:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000040:3.0:1786176991.736938:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053162887] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176991.736946:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856323072, portal 25
00000100:00000001:3.0:1786176991.736947:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.736951:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1786176991.736952:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176991.736956:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000100:00080000:3.0:1786176991.736957:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786176991.736959:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176991.736962:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176991.736965:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786176991.736968:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7215
00000100:00000040:1.0:1786176991.736969:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786176991.736973:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176991.736979:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176991.736982:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176991.736986:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.736987:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176991.736989:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.736990:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786176991.736991:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176991.736992:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786176991.736993:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786176991.736996:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786176991.736999:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786176991.737002:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:1.0:1786176991.737003:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786176991.737006:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176991.737011:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786176991.737011:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:1.0:1786176991.737017:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000040:3.0:1786176991.737017:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.737019:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176991.737022:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000200:1.0:1786176991.737023:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856323072, offset 0
00000100:00080000:3.0:1786176991.737025:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176991.737029:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176991.737030:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786176991.737034:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786176991.737034:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176991.737038:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176991.737040:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176991.737041:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786176991.737043:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176991.737047:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176991.737048:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176991.737052:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c000.
00000400:00000200:1.0:1786176991.737055:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:3.0:1786176991.737056:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786176991.737062:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:1.0:1786176991.737064:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1786176991.737064:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176991.737066:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176991.737068:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1786176991.737070:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:3.0:1786176991.737071:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176991.737075:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000010:3.0:1786176991.737077:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000400:00000200:1.0:1786176991.737079:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.737083:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176991.737086:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254695:254695:256:4294967295] 192.168.202.110@tcp LPNI seq info [254684:254684:8:4294967295]
00000100:00100000:3.0:1786176991.737091:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176991.737095:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786176991.737102:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176991.737106:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176991.737109:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006fe8fdb7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176991.737118:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176991.737121:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786176991.737122:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1786176991.737126:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786176991.737129:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:1.0:1786176991.737132:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786176991.737132:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176991.737135:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:1.0:1786176991.737139:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000100:00080000:3.0:1786176991.737139:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7215
00000400:00000200:3.0:1786176991.737145:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176991.737151:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:1.0:1786176991.737152:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786176991.737153:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:1.0:1786176991.737157:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000100:00000040:3.0:1786176991.737158:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.737162:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176991.737166:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.737171:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:1.1:1786176991.737171:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:3.0:1786176991.737175:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176991.737178:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786176991.737179:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:1.1:1786176991.737179:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000020:00000040:3.0:1786176991.737182:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786176991.737186:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786176991.737188:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000100:1.0:1786176991.737193:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786176991.737194:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.737198:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:1.0:1786176991.737199:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000100:00000040:3.0:1786176991.737200:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.737205:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786176991.737207:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786176991.737208:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1786176991.737210:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1786176991.737211:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176991.737215:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786176991.737218:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176991.737222:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176991.737223:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1786176991.737224:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176991.737228:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:1.0:1786176991.737231:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000008:00000001:3.0:1786176991.737231:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176991.737237:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:3.0:1786176991.737237:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786176991.737242:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c700.
00000100:00000001:1.0:1786176991.737244:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:3.0:1786176991.737245:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176991.737248:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:1.0:1786176991.737251:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176991.737251:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176991.737252:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176991.737255:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786176991.737257:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786176991.737262:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:3.0:1786176991.737267:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.737268:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00100000:3.0:1786176991.737273:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1786176991.737275:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786176991.737291:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176991.737295:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.737300:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786176991.737301:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e56458ae] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737308:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786176991.737313:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176991.737315:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176991.737318:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786176991.737318:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786176991.737320:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786176991.737323:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176991.737325:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786176991.737328:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1786176991.737329:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7215
00000100:00000001:1.0:1786176991.737335:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301891277
00000400:00000200:3.0:1786176991.737335:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786176991.737340:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786176991.737342:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786176991.737344:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000010:1.0:1786176991.737346:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1420.
00000100:00000001:3.0:1786176991.737346:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176991.737351:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786176991.737355:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176991.737356:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000001:3.0:1786176991.737358:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786176991.737361:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786176991.737363:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786176991.737365:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301891277
00000020:00000040:3.0:1786176991.737366:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786176991.737372:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000010:1.0:1786176991.737373:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1c00.
00000020:00000001:3.0:1786176991.737377:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176991.737380:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:3.0:1786176991.737385:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176991.737387:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786176991.737388:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786176991.737392:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176991.737394:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176991.737395:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176991.737401:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000020:00000040:3.0:1786176991.737402:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786176991.737407:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786176991.737409:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786176991.737412:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:1.0:1786176991.737413:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176991/real 1786176991]  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786176991.737414:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786176991.737418:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176991.737421:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786176991.737424:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786176991.737428:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176991.737431:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176991.737436:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000010:3.0:1786176991.737437:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e300.
00000400:00000001:1.0:1786176991.737441:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:3.0:1786176991.737442:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786176991.737446:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:1.0:1786176991.737447:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
02000000:00000001:3.0:1786176991.737448:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786176991.737452:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786176991.737454:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1786176991.737455:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
02000000:00000001:3.0:1786176991.737456:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786176991.737461:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
02000000:00000010:3.0:1786176991.737462:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:1.0:1786176991.737467:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:3.0:1786176991.737469:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176991.737475:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786176991.737479:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786176991.737496:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786176991.737498:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176991.737507:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786176991.737539:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e3752a60] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737543:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176991.737547:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176991.737551:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.737552:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176991.737557:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786176991.737557:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7225)
00000100:00100000:1.0:1786176991.737568:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737578:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176991.737582:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737587:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176991.737590:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.737591:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176991.737592:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.737594:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176991.737595:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176991.737597:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176991.737599:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176991.737601:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176991.737604:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176991.737610:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856323072:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786176991.737614:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.737615:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176991.737618:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856323072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737622:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176991.737624:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176991.737626:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176991.737629:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176991.737630:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176991.737631:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786176991.737632:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176991.737634:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786176991.737638:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176991.737639:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176991.737640:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:1.0:1786176991.737643:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176991.737644:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.737645:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.737651:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176991.737652:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.737653:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176991.737655:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176991.737656:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176991.737658:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737663:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176991.737664:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176991.737665:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176991.737667:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176991.737668:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176991.737670:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856323200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176991.737672:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176991.737673:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176991.737675:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176991.737676:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176991.737678:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176991.737680:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176991.737683:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176991.737685:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176991.737688:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176991.737690:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856323200, portal 10
00000100:00000001:1.0:1786176991.737691:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176991.737692:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:1.0:1786176991.737697:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737702:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176991.737703:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176991.737705:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176991.737706:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856323200, offset 0
00000400:00000200:1.0:1786176991.737709:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176991.737712:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176991.737715:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176991.737717:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176991.737719:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176991.737721:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176991.737722:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176991.737724:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254696:254696:256:4294967295] 192.168.202.110@tcp LPNI seq info [254685:254685:8:4294967295]
00000400:00000200:1.0:1786176991.737727:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176991.737731:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176991.737734:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176991.737737:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176991.737739:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176991.737742:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786176991.737745:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176991.737761:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1786176991.737764:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176991.737765:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176991.737768:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176991.737771:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176991.737773:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176991.737774:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176991.737776:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176991.737779:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737784:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176991.737786:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737793:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.737795:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176991.737798:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176991.737803:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176991.737807:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176991.737809:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301887498
00000100:00000001:1.0:1786176991.737811:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301891277
00000100:00000001:1.0:1786176991.737814:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301887498
00000100:00000001:1.0:1786176991.737816:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301891277
00000100:00000001:1.0:1786176991.737819:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176991.737821:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.737823:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176991.737828:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176991/real 1786176991]  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176991.737837:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176991.737838:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176991.737840:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176991.737843:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176991.737845:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176991.737847:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176991.737850:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176991.737855:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737859:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176991.737860:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.737862:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176991.737863:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176991.737865:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737869:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176991.737872:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737876:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176991.737878:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.737879:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176991.737879:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.737881:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176991.737882:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176991.737883:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176991.737885:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176991.737887:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176991.737889:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176991.737894:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856323200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176991.737896:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176991.737898:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737907:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176991.737908:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176991.737910:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176991.737911:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176991.737913:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176991.737914:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856323328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176991.737916:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176991.737918:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176991.737919:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176991.737920:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176991.737922:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176991.737924:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176991.737926:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176991.737928:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176991.737930:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176991.737932:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856323328, portal 10
00000100:00000001:1.0:1786176991.737933:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176991.737934:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:1.0:1786176991.737938:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.737943:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176991.737944:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176991.737946:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176991.737947:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856323328, offset 0
00000400:00000200:1.0:1786176991.737951:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176991.737956:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176991.737959:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176991.737962:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176991.737964:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176991.737967:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176991.737969:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176991.737973:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254697:254697:256:4294967295] 192.168.202.110@tcp LPNI seq info [254686:254686:8:4294967295]
00000400:00000200:1.0:1786176991.737977:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176991.737985:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176991.737990:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176991.737994:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176991.737998:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176991.738001:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786176991.738005:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176991.738007:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1786176991.738010:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176991.738011:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176991.738013:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176991.738016:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176991.738018:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176991.738019:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176991.738021:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176991.738023:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738029:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176991.738031:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738036:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.738037:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176991.738039:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176991.738042:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176991.738044:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176991.738045:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301887623
00000100:00000001:1.0:1786176991.738047:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301891277
00000100:00000001:1.0:1786176991.738048:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301887623
00000100:00000001:1.0:1786176991.738050:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301891277
00000100:00000001:1.0:1786176991.738051:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176991.738053:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.738055:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176991.738057:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176991/real 1786176991]  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176991.738062:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176991.738063:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176991.738064:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176991.738065:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176991.738067:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176991.738068:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176991.738070:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176991.738076:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738079:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176991.738080:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738081:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176991.738082:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176991.738084:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738088:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176991.738091:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738095:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176991.738096:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738097:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176991.738098:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738100:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176991.738101:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176991.738103:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176991.738105:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176991.738107:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176991.738109:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176991.738114:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856323328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176991.738116:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176991.738118:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738122:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176991.738123:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176991.738124:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176991.738125:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176991.738127:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176991.738129:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856323456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176991.738131:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176991.738132:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176991.738133:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176991.738134:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176991.738137:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176991.738138:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176991.738140:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176991.738141:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176991.738143:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176991.738145:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856323456, portal 4
00000100:00000001:1.0:1786176991.738147:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176991.738147:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:1.0:1786176991.738152:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738156:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176991.738157:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176991.738159:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176991.738160:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856323456, offset 0
00000400:00000200:1.0:1786176991.738163:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176991.738165:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176991.738168:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176991.738170:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176991.738172:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176991.738175:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176991.738177:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176991.738180:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254698:254698:256:4294967295] 192.168.202.110@tcp LPNI seq info [254687:254687:8:4294967295]
00000400:00000200:1.0:1786176991.738185:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176991.738191:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176991.738196:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176991.738199:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176991.738203:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176991.738206:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786176991.738211:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176991.738214:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1786176991.738218:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176991.738220:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176991.738224:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176991.738227:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176991.738229:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176991.738230:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176991.738232:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176991.738235:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738240:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176991.738242:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738247:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.738248:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176991.738250:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176991.738253:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176991.738255:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176991.738256:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301887498
00000100:00000001:1.0:1786176991.738258:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301891277
00000100:00000001:1.0:1786176991.738259:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301887498
00000100:00000001:1.0:1786176991.738261:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301891277
00000100:00000001:1.0:1786176991.738262:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176991.738263:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.738265:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176991.738267:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176991/real 1786176991]  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176991.738272:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176991.738273:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176991.738274:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176991.738275:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176991.738276:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176991.738278:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176991.738280:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176991.738285:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738289:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176991.738290:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738291:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176991.738292:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176991.738294:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738298:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176991.738300:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738305:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176991.738306:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738307:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176991.738309:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738310:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176991.738311:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176991.738312:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176991.738314:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176991.738315:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176991.738318:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176991.738322:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856323456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176991.738324:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176991.738326:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738330:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176991.738331:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176991.738332:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176991.738333:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176991.738335:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176991.738336:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856323584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176991.738338:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176991.738340:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176991.738341:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176991.738342:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176991.738344:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176991.738345:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176991.738347:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176991.738349:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176991.738351:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176991.738353:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856323584, portal 4
00000100:00000001:1.0:1786176991.738354:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176991.738355:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:1.0:1786176991.738359:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738363:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176991.738364:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176991.738366:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176991.738367:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856323584, offset 0
00000400:00000200:1.0:1786176991.738370:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176991.738372:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176991.738374:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176991.738376:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176991.738378:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176991.738379:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176991.738381:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176991.738383:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254699:254699:256:4294967295] 192.168.202.110@tcp LPNI seq info [254688:254688:8:4294967295]
00000400:00000200:1.0:1786176991.738386:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176991.738389:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176991.738392:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176991.738394:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176991.738397:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176991.738399:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1786176991.738402:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176991.738403:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1786176991.738406:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176991.738407:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176991.738409:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176991.738411:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176991.738413:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176991.738414:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176991.738416:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176991.738418:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738423:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176991.738425:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177016 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738429:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.738431:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176991.738433:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176991.738434:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176991.738436:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176991.738437:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301887506
00000100:00000001:1.0:1786176991.738439:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301891277
00000100:00000001:1.0:1786176991.738440:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301887506
00000100:00000001:1.0:1786176991.738442:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301891277
00000100:00000001:1.0:1786176991.738443:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176991.738444:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.738445:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176991.738447:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176991/real 1786176991]  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176991.738452:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176991.738452:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176991.738453:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176991.738454:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176991.738455:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176991.738457:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176991.738458:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176991.738463:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738471:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176991.738472:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738473:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176991.738474:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176991.738477:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738481:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176991.738483:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738487:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176991.738488:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738489:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176991.738490:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738491:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176991.738492:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176991.738493:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176991.738495:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176991.738496:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176991.738499:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176991.738503:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856323584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176991.738505:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738506:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176991.738508:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856323200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738532:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176991.738534:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176991.738536:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786176991.738539:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176991.738540:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176991.738542:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176991.738544:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176991.738546:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786176991.738548:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176991.738550:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176991.738551:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:1.0:1786176991.738554:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176991.738556:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738558:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176991.738562:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856323328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738569:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176991.738571:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176991.738572:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176991.738575:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176991.738576:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176991.738577:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176991.738580:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176991.738581:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:1.0:1786176991.738582:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176991.738583:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176991.738584:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:1.0:1786176991.738586:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176991.738587:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738588:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176991.738590:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856323456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738595:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176991.738596:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176991.738597:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176991.738598:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176991.738599:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176991.738600:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176991.738601:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176991.738602:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:1.0:1786176991.738603:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176991.738604:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176991.738605:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:1.0:1786176991.738606:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176991.738607:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738608:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176991.738610:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856323584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177016 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176991.738614:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176991.738614:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176991.738615:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176991.738617:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176991.738618:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176991.738619:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176991.738620:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176991.738621:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786176991.738622:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176991.738622:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176991.738623:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:1.0:1786176991.738624:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176991.738625:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738626:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176991.738629:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176991.738630:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176991.738631:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176991.738632:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176992.312632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176992.312638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176992.312642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176992.312643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176992.312645:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176992.312646:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176992.312699:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176992.312703:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176992.312706:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176992.760616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176992.760622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176992.760623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176992.760627:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176992.760630:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176992.760633:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176992.760635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176992.760637:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176992.760639:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176993.336625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176993.336632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176993.336638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176993.336640:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176993.336644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176993.336646:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786176993.336649:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786176993.336652:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176993.336655:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786176993.336658:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176993.336660:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176993.336662:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176993.336713:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176993.336717:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176993.336720:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176993.784582:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176993.784589:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176993.784591:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176993.784596:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176993.784600:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176993.784604:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176993.784607:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176993.784610:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176993.784612:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176994.360630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176994.360637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176994.360644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176994.360646:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176994.360650:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176994.360651:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176994.360762:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176994.360766:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176994.360769:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786176994.808620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176994.808627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176994.808628:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176994.808633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176994.808636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176994.808640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176994.808642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176994.808644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176994.808646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176995.384590:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176995.384595:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176995.384599:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176995.384600:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176995.384603:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176995.384604:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176995.384678:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176995.384683:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176995.384687:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786176995.576600:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301891277, at 4301891328
00000100:00000040:0.0:1786176995.576601:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301891277, at 4301891328
00000100:00000010:0.0:1786176995.576615:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1420.
00000100:00000010:1.0:1786176995.576617:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1c00.
00000100:00000001:2.0:1786176995.832624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786176995.832630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786176995.832632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176995.832636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786176995.832639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176995.832642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176995.832644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786176995.832646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176995.832648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176996.408664:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176996.408671:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176996.408678:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176996.408680:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176996.408684:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176996.408685:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176996.408748:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176996.408752:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176996.408754:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786176996.856613:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786176996.856614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176996.856620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176996.856622:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176996.856624:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786176996.856628:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1786176996.856628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786176996.856632:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786176996.856632:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786176996.856637:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786176996.856637:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176996.856641:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176996.856644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786176996.856646:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7220
00000100:00000001:3.0:1786176996.856648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786176996.856657:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176996.856663:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176996.856666:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176996.856669:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.856671:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176996.856674:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.856676:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176996.856679:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176996.856680:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786176996.856682:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176996.856685:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786176996.856686:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786176996.856688:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786176996.856691:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.856692:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176996.856694:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.856695:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176996.856711:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786176996.856714:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176996.856716:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176996.856718:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176996.856721:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176996.856723:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176996.856724:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786176996.856726:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176996.856734:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000020:00000040:2.0:1786176996.856737:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786176996.856740:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176996.856741:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176996.856743:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176996.856746:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176996.856749:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176996.856755:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786176996.856760:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176996.856768:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786176996.856776:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176996.856779:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176996.856783:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176996.856802:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176996.856804:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176996.856807:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786176996.856808:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176996.856810:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176996.856812:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176996.856813:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7220
00000400:00000200:2.0:1786176996.856816:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176996.856818:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176996.856820:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176996.856822:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.856824:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176996.856826:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.856828:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176996.856830:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176996.856831:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786176996.856833:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176996.856834:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:1.0:1786176996.856835:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1786176996.856836:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176996.856840:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1786176996.856840:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786176996.856844:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786176996.856845:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.856847:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176996.856851:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:1.0:1786176996.856853:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176996.856853:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176996.856855:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786176996.856860:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176996.856865:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176996.856868:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176996.856873:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1786176996.856874:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176996.856876:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176996.856877:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176996.856882:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786176996.856883:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:2.0:1786176996.856885:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786176996.856888:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176996.856889:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786176996.856890:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176996.856897:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176996.856897:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000020:00000040:2.0:1786176996.856901:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786176996.856903:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786176996.856904:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176996.856906:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176996.856908:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786176996.856912:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176996.856916:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00100000:1.0:1786176996.856919:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856323712:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786176996.856922:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.856928:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176996.856933:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:2.0:1786176996.856934:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786176996.856936:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176996.856941:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786176996.856948:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000010:1.0:1786176996.856949:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176996.856953:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176996.856954:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176996.856960:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786176996.856963:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1786176996.856965:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176996.856973:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000001:2.0:1786176996.856977:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176996.856980:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856323712, portal 25
00000100:00080000:2.0:1786176996.856982:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786176996.856984:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176996.856986:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454188800 : -114548255362816 : ffff97d1a7edf100)
00000100:00080000:2.0:1786176996.856990:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786176996.856993:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:1.0:1786176996.856995:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176996.856996:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176996.857002:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176996.857007:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7220
00000100:00000001:1.0:1786176996.857008:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176996.857014:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786176996.857016:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176996.857017:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176996.857022:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856323712, offset 0
00000400:00000200:2.0:1786176996.857025:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786176996.857029:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786176996.857030:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786176996.857036:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.857039:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786176996.857041:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786176996.857045:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.857048:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176996.857051:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:2.0:1786176996.857053:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176996.857056:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000200:1.0:1786176996.857057:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:2.0:1786176996.857059:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786176996.857063:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:2.0:1786176996.857064:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:1.0:1786176996.857069:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:2.0:1786176996.857069:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786176996.857074:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000001:2.0:1786176996.857075:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786176996.857079:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254700:254700:256:4294967295] 192.168.202.110@tcp LPNI seq info [254689:254689:8:4294967295]
00000100:00000040:2.0:1786176996.857083:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786176996.857087:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786176996.857089:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176996.857093:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.857097:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786176996.857101:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:2.0:1786176996.857103:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000800:00000200:1.0:1786176996.857108:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786176996.857110:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176996.857115:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786176996.857116:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
02000000:00000001:2.0:1786176996.857117:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786176996.857122:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176996.857127:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786176996.857129:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786176996.857130:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176996.857135:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000800:00000200:1.0:1786176996.857136:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000008:00000001:2.0:1786176996.857140:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176996.857144:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786176996.857146:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:2.0:1786176996.857150:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000800:00000200:1.1:1786176996.857152:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000020:00000040:2.0:1786176996.857155:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000800:00000100:1.0:1786176996.857162:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1786176996.857163:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000010:1.0:1786176996.857167:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
02000000:00000001:2.0:1786176996.857167:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176996.857172:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176996.857175:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1786176996.857177:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1786176996.857178:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176996.857183:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000010:2.0:1786176996.857185:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000400:00000200:1.0:1786176996.857188:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176996.857191:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:2.0:1786176996.857192:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.857195:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:2.0:1786176996.857201:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1786176996.857202:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.857230:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786176996.857232:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1786176996.857235:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176996.857238:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.857246:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786176996.857248:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.857249:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176996.857255:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176996.857257:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176996.857260:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786176996.857262:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786176996.857267:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1786176996.857268:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301896397
00000100:00000001:2.0:1786176996.857271:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786176996.857273:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000010:1.0:1786176996.857275:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1c00.
00000100:00000001:2.0:1786176996.857276:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786176996.857280:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7220
00000100:00000001:1.0:1786176996.857283:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000400:00000200:2.0:1786176996.857283:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786176996.857290:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786176996.857291:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301896397
00000100:00080000:2.0:1786176996.857292:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786176996.857296:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000010:1.0:1786176996.857297:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1420.
00000100:00000001:2.0:1786176996.857298:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176996.857302:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:2.0:1786176996.857302:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176996.857307:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786176996.857307:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786176996.857309:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786176996.857311:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786176996.857312:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:1.0:1786176996.857314:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000020:00000001:2.0:1786176996.857315:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786176996.857319:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786176996.857322:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00100000:1.0:1786176996.857323:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176996/real 1786176996]  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786176996.857326:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786176996.857333:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.857338:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786176996.857341:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.857344:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786176996.857349:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786176996.857354:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000001:1.0:1786176996.857355:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786176996.857357:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786176996.857358:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:2.0:1786176996.857360:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:1.0:1786176996.857361:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176996.857365:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
02000000:00000001:2.0:1786176996.857365:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:1.0:1786176996.857369:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
02000000:00000001:2.0:1786176996.857370:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.857372:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1786176996.857373:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000200:1.0:1786176996.857378:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1786176996.857378:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176996.857384:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176996.857389:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176996.857393:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786176996.857394:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786176996.857399:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786176996.857404:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786176996.857406:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:2.0:1786176996.857406:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.857409:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176996.857409:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176996.857413:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176996.857414:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176996.857417:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786176996.857417:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176996.857424:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00100000:1.0:1786176996.857425:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176996.857430:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.857436:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786176996.857439:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1786176996.857443:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.857448:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176996.857451:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.857455:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176996.857457:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.857461:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176996.857464:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786176996.857464:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786176996.857468:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786176996.857469:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176996.857474:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000040:2.0:1786176996.857475:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.857477:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176996.857485:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176996.857489:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176996.857494:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786176996.857499:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00100000:1.0:1786176996.857500:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856323712:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786176996.857502:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786176996.857506:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786176996.857506:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786176996.857508:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786176996.857531:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176996.857536:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf100 x1872934856323712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786176996.857539:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7220
00000400:00000200:2.0:1786176996.857548:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176996.857550:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000400:00000200:2.0:1786176996.857553:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:1.0:1786176996.857554:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00080000:2.0:1786176996.857556:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
02000000:00000010:1.0:1786176996.857557:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
00000100:00000040:2.0:1786176996.857563:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:1.0:1786176996.857564:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:2.0:1786176996.857567:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000020:00000001:1.0:1786176996.857568:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176996.857570:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000040:2.0:1786176996.857571:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000020:00000001:1.0:1786176996.857574:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00000001:2.0:1786176996.857575:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000010:1.0:1786176996.857578:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
00000020:00000001:2.0:1786176996.857579:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786176996.857581:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786176996.857583:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
02000000:00000001:1.0:1786176996.857584:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000020:00000040:2.0:1786176996.857585:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
02000000:00000001:1.0:1786176996.857588:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000020:00000001:2.0:1786176996.857589:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000010:1.0:1786176996.857591:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf100.
00000020:00000001:2.0:1786176996.857592:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786176996.857596:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000040:2.0:1786176996.857596:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786176996.857598:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786176996.857600:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786176996.857602:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786176996.857604:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786176996.857607:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786176996.857609:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000020:00000040:2.0:1786176996.857609:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:1.0:1786176996.857613:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786176996.857613:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786176996.857618:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.857619:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:2.0:1786176996.857622:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786176996.857623:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176996.857626:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:2.0:1786176996.857626:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176996.857630:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176996.857632:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786176996.857633:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786176996.857636:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786176996.857641:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786176996.857645:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.857647:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176996.857650:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786176996.857650:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
02000000:00000001:1.0:1786176996.857654:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:2.0:1786176996.857656:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:1.0:1786176996.857657:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786176996.857660:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786176996.857661:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786176996.857664:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176996.857665:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856323840:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786176996.857667:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786176996.857670:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786176996.857672:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786176996.857673:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176996.857675:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176996.857677:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786176996.857677:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
02000000:00000001:1.0:1786176996.857682:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786176996.857683:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786176996.857686:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176996.857690:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786176996.857691:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786176996.857709:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176996.857715:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000100:00000001:2.0:1786176996.857717:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000010:1.0:1786176996.857719:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000001:2.0:1786176996.857721:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786176996.857724:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856323840, portal 10
00000100:00000040:2.0:1786176996.857727:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.857728:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176996.857730:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00000001:2.0:1786176996.857741:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176996.857744:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786176996.857746:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7230)
00000100:00000001:1.0:1786176996.857756:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176996.857759:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176996.857763:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176996.857766:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856323840, offset 0
00000400:00000200:1.0:1786176996.857770:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176996.857774:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176996.857778:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176996.857781:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176996.857783:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176996.857786:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176996.857788:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176996.857791:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254701:254701:256:4294967295] 192.168.202.110@tcp LPNI seq info [254690:254690:8:4294967295]
00000400:00000200:1.0:1786176996.857796:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176996.857802:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176996.857807:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176996.857811:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176996.857814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176996.857818:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786176996.857822:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176996.857824:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1786176996.857828:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176996.857830:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176996.857833:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176996.857837:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176996.857840:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176996.857841:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176996.857843:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176996.857846:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.857852:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176996.857854:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.857858:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.857859:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176996.857862:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176996.857865:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786176996.857867:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176996.857870:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301892618
00000100:00000001:1.0:1786176996.857873:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301896397
00000100:00000001:1.0:1786176996.857875:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301892618
00000100:00000001:1.0:1786176996.857877:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301896397
00000100:00000001:1.0:1786176996.857880:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176996.857882:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.857884:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176996.857888:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176996/real 1786176996]  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176996.857896:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176996.857898:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176996.857900:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176996.857903:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176996.857905:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176996.857907:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176996.857911:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176996.857920:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.857926:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176996.857927:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.857929:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176996.857930:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176996.857932:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.857937:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176996.857939:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.857944:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176996.857945:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.857946:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176996.857947:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.857949:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176996.857950:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176996.857952:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176996.857954:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176996.857955:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176996.857958:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176996.857962:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856323840:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176996.857966:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176996.857968:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.857972:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176996.857973:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176996.857975:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176996.857976:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176996.857977:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176996.857979:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856323968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176996.857981:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176996.857982:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176996.857984:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176996.857985:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176996.857988:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176996.857989:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176996.857991:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176996.857993:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176996.857995:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176996.857997:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856323968, portal 10
00000100:00000001:1.0:1786176996.857999:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176996.857999:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:1.0:1786176996.858003:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858008:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176996.858009:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176996.858011:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176996.858012:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856323968, offset 0
00000400:00000200:1.0:1786176996.858015:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176996.858017:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176996.858019:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176996.858022:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176996.858024:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176996.858025:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176996.858027:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176996.858029:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254702:254702:256:4294967295] 192.168.202.110@tcp LPNI seq info [254691:254691:8:4294967295]
00000400:00000200:1.0:1786176996.858032:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176996.858036:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176996.858038:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176996.858040:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176996.858043:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176996.858044:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786176996.858047:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176996.858049:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1786176996.858051:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176996.858053:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176996.858055:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176996.858057:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176996.858059:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176996.858060:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176996.858062:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176996.858064:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858069:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176996.858071:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858076:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.858077:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176996.858078:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176996.858081:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786176996.858083:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176996.858084:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301892743
00000100:00000001:1.0:1786176996.858086:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301896397
00000100:00000001:1.0:1786176996.858087:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301892743
00000100:00000001:1.0:1786176996.858089:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301896397
00000100:00000001:1.0:1786176996.858090:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176996.858092:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.858093:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176996.858095:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176996/real 1786176996]  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176996.858100:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176996.858101:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176996.858102:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176996.858103:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176996.858104:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176996.858105:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176996.858107:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176996.858113:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858120:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176996.858121:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858123:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176996.858124:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176996.858128:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858135:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176996.858139:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858146:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176996.858149:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858151:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176996.858152:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858155:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176996.858157:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176996.858160:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176996.858163:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176996.858166:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176996.858170:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176996.858176:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856323968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786176996.858178:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176996.858181:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858185:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176996.858186:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176996.858188:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176996.858189:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176996.858190:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176996.858192:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856324096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176996.858194:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176996.858195:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176996.858196:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176996.858198:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176996.858200:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176996.858201:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176996.858204:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176996.858206:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176996.858208:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176996.858210:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856324096, portal 4
00000100:00000001:1.0:1786176996.858212:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176996.858212:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:1.0:1786176996.858216:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858221:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176996.858222:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176996.858224:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176996.858225:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856324096, offset 0
00000400:00000200:1.0:1786176996.858228:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176996.858230:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176996.858233:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176996.858234:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176996.858236:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176996.858238:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176996.858239:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176996.858241:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254703:254703:256:4294967295] 192.168.202.110@tcp LPNI seq info [254692:254692:8:4294967295]
00000400:00000200:1.0:1786176996.858244:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176996.858248:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176996.858251:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176996.858253:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176996.858255:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176996.858257:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786176996.858260:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176996.858262:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1786176996.858264:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176996.858265:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176996.858268:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176996.858270:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176996.858271:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176996.858273:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176996.858274:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176996.858276:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858282:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176996.858284:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858292:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.858293:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176996.858295:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176996.858297:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786176996.858300:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176996.858301:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301892618
00000100:00000001:1.0:1786176996.858302:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301896397
00000100:00000001:1.0:1786176996.858304:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301892618
00000100:00000001:1.0:1786176996.858306:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301896397
00000100:00000001:1.0:1786176996.858307:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176996.858308:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.858309:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176996.858311:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176996/real 1786176996]  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176996.858316:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176996.858317:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176996.858318:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176996.858320:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176996.858321:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176996.858322:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176996.858324:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176996.858329:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858333:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176996.858334:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858335:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176996.858336:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176996.858338:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858343:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176996.858346:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858350:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176996.858352:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858353:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176996.858353:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858354:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176996.858355:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176996.858357:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176996.858359:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176996.858360:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176996.858362:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176996.858367:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856324096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176996.858369:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858370:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176996.858372:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856323840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858376:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176996.858377:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176996.858378:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786176996.858380:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176996.858381:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176996.858382:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176996.858384:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176996.858385:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:1.0:1786176996.858387:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176996.858388:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176996.858389:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:1.0:1786176996.858391:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176996.858392:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858393:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176996.858395:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856323968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858400:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176996.858400:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176996.858401:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786176996.858403:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176996.858403:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176996.858404:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786176996.858405:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176996.858406:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:1.0:1786176996.858407:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176996.858408:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176996.858409:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:1.0:1786176996.858410:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176996.858411:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858412:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176996.858414:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856324096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858417:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176996.858418:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176996.858419:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176996.858420:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176996.858420:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176996.858421:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176996.858422:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176996.858423:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:1.0:1786176996.858425:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176996.858426:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176996.858427:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:1.0:1786176996.858428:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176996.858429:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858431:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858433:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176996.858434:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.858435:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176996.858436:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786176996.858437:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786176996.858439:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858443:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786176996.858444:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176996.858446:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786176996.858447:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786176996.858448:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786176996.858450:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856324224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176996.858452:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786176996.858453:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786176996.858454:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786176996.858455:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786176996.858457:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176996.858458:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786176996.858460:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786176996.858461:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000400:00000010:1.0:1786176996.858463:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00000200:1.0:1786176996.858465:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856324224, portal 4
00000100:00000001:1.0:1786176996.858466:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786176996.858467:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:1.0:1786176996.858471:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858475:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786176996.858477:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786176996.858478:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
00000100:00000200:1.0:1786176996.858479:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856324224, offset 0
00000400:00000200:1.0:1786176996.858482:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786176996.858484:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176996.858487:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786176996.858488:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786176996.858490:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786176996.858492:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786176996.858493:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786176996.858495:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254704:254704:256:4294967295] 192.168.202.110@tcp LPNI seq info [254693:254693:8:4294967295]
00000400:00000200:1.0:1786176996.858498:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786176996.858502:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786176996.858504:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176996.858506:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786176996.858508:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786176996.858526:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1786176996.858529:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786176996.858530:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1786176996.858533:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786176996.858535:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:1.0:1786176996.858537:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786176996.858539:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786176996.858541:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:1.0:1786176996.858542:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:1.0:1786176996.858544:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786176996.858546:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858552:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176996.858553:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177021 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858558:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.858559:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786176996.858561:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786176996.858562:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786176996.858565:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786176996.858566:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301892627
00000100:00000001:1.0:1786176996.858568:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301896397
00000100:00000001:1.0:1786176996.858569:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301892627
00000100:00000001:1.0:1786176996.858571:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301896397
00000100:00000001:1.0:1786176996.858572:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786176996.858573:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.858574:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786176996.858577:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176996/real 1786176996]  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786176996.858581:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786176996.858582:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786176996.858583:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786176996.858585:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786176996.858586:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786176996.858587:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786176996.858589:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786176996.858595:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858598:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786176996.858599:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858600:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176996.858601:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786176996.858603:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858607:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786176996.858609:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858613:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786176996.858615:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858616:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786176996.858617:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858618:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786176996.858619:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786176996.858621:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786176996.858623:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786176996.858624:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786176996.858626:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786176996.858631:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856324224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786176996.858633:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858634:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786176996.858636:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856324224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177021 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786176996.858640:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786176996.858641:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786176996.858643:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786176996.858644:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786176996.858645:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786176996.858645:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786176996.858647:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786176996.858648:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:1.0:1786176996.858649:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786176996.858650:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786176996.858651:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:1.0:1786176996.858653:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786176996.858653:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858655:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786176996.858656:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786176996.858657:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786176996.858659:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786176996.858660:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176997.432616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176997.432622:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176997.432626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176997.432627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176997.432630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176997.432631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786176997.432684:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786176997.432688:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786176997.432711:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176997.880632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176997.880640:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176997.880642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176997.880649:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176997.880652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176997.880657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176997.880660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176997.880663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176997.880666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176998.456625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176998.456631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176998.456635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176998.456636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176998.456638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176998.456639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786176998.456701:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786176998.456705:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786176998.456708:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176998.904621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176998.904628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176998.904630:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176998.904635:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176998.904638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176998.904641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176998.904643:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176998.904646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176998.904648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176999.480622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176999.480627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176999.480631:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176999.480632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786176999.480634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786176999.480635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786176999.480707:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786176999.480712:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786176999.480716:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786176999.928592:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786176999.928601:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786176999.928603:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786176999.928609:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786176999.928613:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786176999.928616:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176999.928619:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786176999.928622:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786176999.928624:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177000.504649:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177000.504657:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177000.504686:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177000.504687:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177000.504690:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177000.504691:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177000.504793:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177000.504800:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177000.504806:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177000.696673:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301896397, at 4301896448
00000100:00000040:1.0:1786177000.696677:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301896397, at 4301896448
00000100:00000010:0.0:1786177000.696690:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1c00.
00000100:00000010:1.0:1786177000.696695:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1420.
00000100:00000001:3.0:1786177000.952579:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177000.952587:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177000.952589:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177000.952595:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177000.952599:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177000.952603:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177000.952606:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177000.952609:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177000.952612:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177001.528703:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177001.528715:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177001.528718:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177001.528722:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177001.528728:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177001.528731:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177001.528733:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177001.528736:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177001.528740:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177001.528742:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177001.528743:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177001.528746:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177001.528748:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177001.528749:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177001.528751:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177001.528753:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177001.528756:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177001.528758:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177001.528825:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177001.528829:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177001.528832:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177001.976609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177001.976615:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177001.976617:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177001.976622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177001.976625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177001.976629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177001.976631:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177001.976633:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177001.976636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177001.976694:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177001.976706:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177001.976709:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177001.976714:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177001.976727:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177001.976738:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7225
00000400:00000200:0.0:1786177001.976749:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177001.976757:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177001.976761:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177001.976763:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.976767:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177001.976769:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.976771:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177001.976775:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177001.976777:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177001.976779:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177001.976782:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177001.976784:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177001.976786:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177001.976789:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.976791:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177001.976793:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.976795:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177001.976799:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177001.976803:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177001.976806:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177001.976810:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177001.976815:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177001.976819:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.976822:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177001.976825:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177001.976838:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a300.
00000020:00000040:0.0:1786177001.976843:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177001.976848:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177001.976850:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.976855:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177001.976859:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177001.976865:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177001.976874:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000100:00000001:0.0:1786177001.976881:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177001.976892:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177001.976902:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177001.976906:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177001.976910:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b0d3569] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177001.976936:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177001.976938:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177001.976941:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177001.976942:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177001.976944:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177001.976946:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177001.976948:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7225
00000400:00000200:0.0:1786177001.976951:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177001.976954:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177001.976956:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177001.976958:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.976960:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177001.976963:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.976965:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177001.976967:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177001.976968:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177001.976969:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177001.976970:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177001.976972:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177001.976973:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177001.976975:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177001.976977:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177001.976979:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177001.976984:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.976987:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177001.976988:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1786177001.976990:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177001.976994:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177001.976995:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1786177001.976997:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177001.977000:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177001.977006:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177001.977013:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.977016:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:1.0:1786177001.977016:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1786177001.977025:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177001.977037:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.977045:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177001.977053:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41bb80.
00000100:00000001:1.0:1786177001.977054:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000040:0.0:1786177001.977058:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786177001.977060:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177001.977061:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177001.977062:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.977064:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177001.977064:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177001.977066:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177001.977068:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000001:0.0:1786177001.977071:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177001.977078:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786177001.977081:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1786177001.977089:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1786177001.977089:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177001.977093:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177001.977100:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9983a05] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786177001.977100:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177001.977119:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856324352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177001.977120:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177001.977130:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177001.977138:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1786177001.977141:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1786177001.977142:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177001.977147:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:1.0:1786177001.977149:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1786177001.977150:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177001.977158:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7225
02000000:00000001:1.0:1786177001.977158:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177001.977166:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1786177001.977169:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000400:00000200:0.0:1786177001.977170:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177001.977176:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177001.977178:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.977180:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177001.977181:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.977183:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000010:1.0:1786177001.977184:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
00000020:00000001:0.0:1786177001.977186:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177001.977188:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177001.977193:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
02000000:00000001:1.0:1786177001.977194:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177001.977196:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177001.977202:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177001.977203:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786177001.977209:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177001.977211:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:1.0:1786177001.977218:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d840.
00000100:00000001:0.0:1786177001.977220:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177001.977228:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.977231:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177001.977234:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000010:1.0:1786177001.977235:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c220.
00000100:00080000:0.0:1786177001.977245:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177001.977248:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177001.977248:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856324352, portal 25
02000000:00000001:0.0:1786177001.977258:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786177001.977260:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177001.977272:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761663744 : -114547947887872 : ffff97d1ba41a300)
02000000:00000001:0.0:1786177001.977273:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177001.977287:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.977296:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:1.0:1786177001.977298:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1786177001.977305:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177001.977316:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.977327:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177001.977328:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000010:0.0:1786177001.977335:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419c00.
00000100:00000040:1.0:1786177001.977337:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:0.0:1786177001.977341:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000010:1.0:1786177001.977343:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c440.
02000000:00000001:0.0:1786177001.977353:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:1.0:1786177001.977354:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856324352, offset 0
02000000:00000001:0.0:1786177001.977356:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.977361:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177001.977363:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177001.977365:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177001.977371:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000010:0.0:1786177001.977374:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5800.
00000100:00000001:0.0:1786177001.977378:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177001.977385:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:0.0:1786177001.977386:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177001.977394:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786177001.977398:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1786177001.977400:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177001.977408:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1786177001.977409:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cf890c1c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786177001.977419:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177001.977431:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1786177001.977441:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177001.977441:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177001.977448:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254705:254705:256:4294967295] 192.168.202.110@tcp LPNI seq info [254694:254694:8:4294967295]
00000400:00000200:1.0:1786177001.977458:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177001.977459:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177001.977464:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177001.977465:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177001.977468:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786177001.977478:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177001.977479:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177001.977491:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7225
00000800:00000200:1.0:1786177001.977502:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177001.977543:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a300.
00000800:00000200:1.0:1786177001.977550:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177001.977555:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:0.0:1786177001.977560:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177001.977566:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786177001.977566:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177001.977575:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786177001.977584:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177001.977590:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177001.977591:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301901518
00000100:00000001:0.0:1786177001.977595:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177001.977600:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000010:1.0:1786177001.977600:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1420.
00000100:00000001:0.0:1786177001.977614:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177001.977617:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000001:0.0:1786177001.977619:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177001.977623:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177001.977626:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177001.977629:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:1.0:1786177001.977630:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301901518
00000020:00000001:0.0:1786177001.977636:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000010:1.0:1786177001.977637:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1e40.
00000020:00000001:0.0:1786177001.977641:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786177001.977643:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:0.0:1786177001.977659:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.977665:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177001.977667:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1786177001.977670:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177001.977678:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.977679:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177001.977686:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177001.977687:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:1.0:1786177001.977700:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177001.977701:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177001.977709:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177001.977712:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786177001.977714:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:0.0:1786177001.977715:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177001.977717:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
02000000:00000001:0.0:1786177001.977723:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177001.977724:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177001.977736:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177001.977738:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000008:00100000:0.0:1786177001.977742:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786177001.977744:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000008:00000001:0.0:1786177001.977746:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.977751:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177001.977755:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a680.
00000020:00000040:0.0:1786177001.977760:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000040:1.0:1786177001.977761:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177001.977763:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177001.977766:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.977776:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177001.977779:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177001.977783:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177001.977792:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4c00.
00000100:00000001:1.0:1786177001.977795:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177001.977800:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.977803:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177001.977804:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177001.977808:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177001.977814:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177001.977820:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856324480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177001.977822:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177001.977825:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177001.977827:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177001.977830:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177001.977834:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
00000100:00100000:0.0:1786177001.977838:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786177001.977838:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177001.977850:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177001.977860:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8db58.
00000100:00000001:0.0:1786177001.977863:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177001.977867:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177001.977867:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c3b8.
00000100:00000200:1.0:1786177001.977878:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856324480, portal 10
00000100:00000040:0.0:1786177001.977884:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000911ee1e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177001.977885:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177001.977891:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761670016 : -114547947881600 : ffff97d1ba41bb80)
00000100:00000001:0.0:1786177001.977908:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177001.977909:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177001.977919:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177001.977930:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177001.977935:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177001.977939:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177001.977942:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177001.977949:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7225
00000100:00000001:1.0:1786177001.977954:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:0.0:1786177001.977958:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177001.977964:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177001.977969:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177001.977971:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:0.0:1786177001.977979:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.977982:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:1.0:1786177001.977984:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c4c8.
00000100:00000040:0.0:1786177001.977987:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.977994:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177001.977996:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000200:1.0:1786177001.977997:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856324480, offset 0
00000020:00000040:0.0:1786177001.977999:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177001.978003:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177001.978008:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177001.978011:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:0.0:1786177001.978013:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786177001.978022:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177001.978022:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177001.978030:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177001.978034:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.978036:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177001.978038:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000400:00000200:1.0:1786177001.978041:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177001.978046:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177001.978050:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177001.978053:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177001.978058:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786177001.978058:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:0.0:1786177001.978065:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177001.978066:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1786177001.978073:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:1.0:1786177001.978079:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00100000:0.0:1786177001.978080:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177001.978089:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177001.978090:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177001.978099:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1786177001.978100:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177001.978111:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254706:254706:255:4294967295] 192.168.202.110@tcp LPNI seq info [254695:254695:7:4294967295]
00000100:00000010:0.0:1786177001.978115:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419500.
00000400:00000200:1.0:1786177001.978121:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1786177001.978122:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177001.978133:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177001.978140:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177001.978147:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1786177001.978152:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:0.0:1786177001.978153:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177001.978160:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177001.978170:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
00000800:00000200:1.0:1786177001.978172:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177001.978183:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786177001.978185:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881aa00.
00000100:00100000:0.0:1786177001.978196:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177001.978208:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177001.978210:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786177001.978221:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786177001.978222:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b6718a9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.1:1786177001.978243:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177001.978247:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177001.978259:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177001.978265:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7235)
00000100:00000040:1.0:1786177001.978268:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786177001.978277:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177001.978280:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301897738
00000100:00000001:1.0:1786177001.978283:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301901518
00000100:00000001:1.0:1786177001.978287:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301897738
00000100:00000001:1.0:1786177001.978289:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301901518
00000100:00000001:1.0:1786177001.978292:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177001.978297:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177001.978301:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177001.978309:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177001.978317:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177001.978318:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177001.978321:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177001.978323:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177001.978325:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177001.978328:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856324608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177001.978331:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177001.978335:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177001.978337:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177001.978339:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177001.978344:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177001.978346:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177001.978350:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177001.978353:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d6e0.
00000400:00000010:1.0:1786177001.978357:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8ca18.
00000100:00000200:1.0:1786177001.978360:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856324608, portal 10
00000100:00000001:1.0:1786177001.978362:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177001.978364:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761661952 : -114547947889664 : ffff97d1ba419c00)
00000100:00000040:1.0:1786177001.978370:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177001.978377:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177001.978379:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177001.978381:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8ce58.
00000100:00000200:1.0:1786177001.978384:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856324608, offset 0
00000400:00000200:1.0:1786177001.978388:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177001.978392:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177001.978397:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177001.978400:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177001.978402:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177001.978405:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177001.978407:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177001.978411:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254707:254707:254:4294967295] 192.168.202.110@tcp LPNI seq info [254696:254696:6:4294967295]
00000400:00000200:1.0:1786177001.978416:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177001.978422:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177001.978427:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177001.978431:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a500.
00000800:00000200:1.0:1786177001.978435:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177001.978439:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1786177001.978442:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177001.978445:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177001.978448:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177001.978450:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301897863
00000100:00000001:1.0:1786177001.978454:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301901518
00000100:00000001:1.0:1786177001.978456:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301897863
00000100:00000001:1.0:1786177001.978458:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301901518
00000100:00000001:1.0:1786177001.978461:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177001.978463:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177001.978467:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177001.978469:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177001.978474:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177001.978476:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1786177001.978479:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177001.978481:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177001.978483:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177001.978488:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177001.978496:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177001.978498:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177001.978500:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177001.978503:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177001.978504:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177001.978507:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856324736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177001.978539:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177001.978542:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177001.978544:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177001.978546:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177001.978549:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177001.978552:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177001.978555:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177001.978558:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8d1b8.
00000400:00000010:1.0:1786177001.978562:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177001.978566:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856324736, portal 4
00000100:00000001:1.0:1786177001.978568:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177001.978569:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761664640 : -114547947886976 : ffff97d1ba41a680)
00000100:00000040:1.0:1786177001.978576:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177001.978583:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177001.978585:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177001.978587:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c990.
00000100:00000200:1.0:1786177001.978589:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856324736, offset 0
00000400:00000200:1.0:1786177001.978594:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177001.978597:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177001.978601:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177001.978604:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177001.978606:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177001.978608:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177001.978610:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177001.978613:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254708:254708:253:4294967295] 192.168.202.110@tcp LPNI seq info [254697:254697:5:4294967295]
00000400:00000200:1.0:1786177001.978617:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177001.978623:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177001.978628:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177001.978631:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881a100.
00000800:00000200:1.0:1786177001.978635:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177001.978639:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1786177001.978642:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177001.978645:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177001.978664:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177001.978666:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301897739
00000100:00000001:1.0:1786177001.978668:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301901518
00000100:00000001:1.0:1786177001.978671:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301897739
00000100:00000001:1.0:1786177001.978673:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301901518
00000100:00000001:1.0:1786177001.978675:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177001.978678:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177001.978680:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177001.978685:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177001.978699:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177001.978700:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177001.978703:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177001.978705:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177001.978707:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177001.978710:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856324864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177001.978713:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177001.978715:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177001.978717:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177001.978718:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177001.978721:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177001.978724:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177001.978727:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177001.978729:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177001.978733:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177001.978736:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856324864, portal 4
00000100:00000001:1.0:1786177001.978738:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177001.978739:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761660160 : -114547947891456 : ffff97d1ba419500)
00000100:00000040:1.0:1786177001.978746:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177001.978753:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177001.978755:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177001.978757:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8cbb0.
00000100:00000200:1.0:1786177001.978760:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856324864, offset 0
00000400:00000200:1.0:1786177001.978764:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177001.978769:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177001.978773:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177001.978776:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177001.978778:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177001.978780:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177001.978782:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177001.978786:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254709:254709:252:4294967295] 192.168.202.110@tcp LPNI seq info [254698:254698:4:4294967295]
00000400:00000200:1.0:1786177001.978790:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177001.978795:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177001.978800:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177001.978804:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177001.978808:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177001.978812:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1786177001.978814:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177001.978818:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177001.978821:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177001.978824:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301897747
00000100:00000001:1.0:1786177001.978826:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301901518
00000100:00000001:1.0:1786177001.978828:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301897747
00000100:00000001:1.0:1786177001.978831:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301901518
00000100:00000001:1.0:1786177001.978833:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177001.978835:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177001.978838:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177001.978841:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177001.978844:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177001.978846:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1786177001.978848:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177001.978849:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177001.978851:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177001.978854:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177002.552634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177002.552641:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177002.552657:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177002.552659:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177002.552662:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177002.552666:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177002.552668:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177002.552671:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177002.552675:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177002.552678:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177002.552680:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177002.552682:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177002.552757:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177002.552762:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177002.552766:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177003.000628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177003.000635:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177003.000651:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177003.000656:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177003.000659:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177003.000662:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177003.000664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177003.000666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177003.000669:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177003.576698:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177003.576708:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177003.576714:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177003.576716:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177003.576720:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177003.576721:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177003.576793:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177003.576798:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177003.576803:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177004.024597:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177004.024607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177004.024609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177004.024614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177004.024619:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177004.024622:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177004.024625:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177004.024652:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177004.024656:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177004.600652:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177004.600659:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177004.600665:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177004.600667:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177004.600669:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177004.600670:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177004.600724:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177004.600728:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177004.600731:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177005.048662:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177005.048675:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177005.048678:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177005.048686:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177005.048690:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177005.048696:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177005.048700:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177005.048704:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177005.048708:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:2.0:1786177005.048870:0:2379:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:2.0:1786177005.048913:0:2379:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1786177005.048923:0:2379:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177005.048927:0:2379:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:2.0:1786177005.048931:0:2379:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177005.048936:0:2379:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1786177005.048940:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786177005.048944:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a300.
00000400:00000200:2.0:1786177005.048950:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786177005.048954:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786177005.048961:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c440
00000400:00000010:2.0:1786177005.048964:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c440.
00000100:00000001:2.0:1786177005.048971:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177005.048983:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177005.049011:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177005.049016:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177005.049025:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177005.049027:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1786177005.049039:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786177005.049042:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881aa00.
00000400:00000200:2.0:1786177005.049047:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786177005.049051:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786177005.049055:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c4c8
00000400:00000010:2.0:1786177005.049057:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c4c8.
00000100:00000001:2.0:1786177005.049061:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177005.049066:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049073:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177005.049076:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1786177005.049081:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1786177005.049083:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049087:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1786177005.049094:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177005.049098:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:1.0:1786177005.049101:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177001/real 1786177005]  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:2.0:1786177005.049109:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786177005.049116:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a500.
00000400:00000200:2.0:1786177005.049121:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786177005.049126:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000001:1.0:1786177005.049130:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177005.049134:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177005.049138:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8db58.
00000400:00000200:1.0:1786177005.049143:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c3b8
00000400:00000010:1.0:1786177005.049145:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c3b8.
00000100:00000001:1.0:1786177005.049149:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000400:00000200:2.0:1786177005.049150:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8ce58
00000400:00000010:2.0:1786177005.049155:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8ce58.
00000100:00000200:1.0:1786177005.049157:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177005.049162:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177005.049170:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177005.049176:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177005.049187:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1786177005.049194:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:2.0:1786177005.049196:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049198:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049205:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177005.049209:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177005.049213:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177005.049217:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:1.0:1786177005.049220:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:2.0:1786177005.049228:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:2.0:1786177005.049235:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881a100.
00000400:00000200:2.0:1786177005.049241:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:1.0:1786177005.049242:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1786177005.049249:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000100:00000040:1.0:1786177005.049255:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786177005.049255:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c990
00000400:00000010:2.0:1786177005.049260:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c990.
00000100:00000001:2.0:1786177005.049266:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1786177005.049271:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000200:2.0:1786177005.049275:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049277:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049282:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177005.049285:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049292:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177005.049294:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1786177005.049298:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:2.0:1786177005.049301:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1786177005.049306:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00000001:2.0:1786177005.049313:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177005.049318:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177005.049318:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177005.049325:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000800:00000100:2.0:1786177005.049330:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000100:00000040:1.0:1786177005.049337:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:2.0:1786177005.049337:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:2.0:1786177005.049343:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1786177005.049352:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1786177005.049357:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8cbb0
00000100:00100000:1.0:1786177005.049360:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856324480:192.168.202.110@tcp:38:kworker.0
00000400:00000010:2.0:1786177005.049361:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8cbb0.
00000100:00000001:2.0:1786177005.049366:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1786177005.049372:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000200:2.0:1786177005.049374:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177005.049381:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177001/real 1786177005]  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177005.049387:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177005.049395:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177026 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177005.049397:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177005.049400:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177005.049404:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d6e0.
00000100:00000001:2.0:1786177005.049409:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177005.049410:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8ca18
00000100:00000001:2.0:1786177005.049413:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000010:1.0:1786177005.049416:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8ca18.
00000100:00000001:1.0:1786177005.049421:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177005.049427:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177005.049445:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049453:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177005.049455:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049457:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177005.049458:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177005.049463:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049470:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177005.049475:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049483:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177005.049486:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049487:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177005.049489:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049491:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177005.049493:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177005.049496:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177005.049500:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177005.049503:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177005.049508:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177005.049555:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856324608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177005.049561:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177005.049566:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177001/real 1786177005]  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177005.049582:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177005.049584:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177005.049587:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d840.
00000400:00000200:1.0:1786177005.049590:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c220
00000400:00000010:1.0:1786177005.049592:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c220.
00000100:00000001:1.0:1786177005.049595:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177005.049598:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177005.049606:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049613:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177005.049614:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049616:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177005.049632:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177005.049636:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049644:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177005.049648:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049654:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177005.049657:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049659:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177005.049660:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049662:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177005.049664:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177005.049666:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177005.049670:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177005.049673:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177005.049677:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177005.049686:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856324352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177005.049691:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177005.049694:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177005.049697:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41bb80 x1872934856324480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049706:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177005.049711:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177005.049717:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177005.049723:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177005.049727:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177005.049730:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177005.049733:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177005.049738:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:1.0:1786177005.049743:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177005.049746:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177005.049749:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41bb80.
00000100:00000001:1.0:1786177005.049754:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177005.049755:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049757:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177005.049761:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419c00 x1872934856324608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049768:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177005.049770:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177005.049771:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177005.049777:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177005.049779:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177005.049781:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177005.049784:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177005.049786:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5800.
02000000:00000001:1.0:1786177005.049789:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177005.049790:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177005.049792:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419c00.
00000100:00000001:1.0:1786177005.049796:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177005.049798:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049800:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177005.049804:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a300 x1872934856324352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049812:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177005.049814:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177005.049816:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177005.049819:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177005.049820:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177005.049822:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786177005.049824:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177005.049826:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:1.0:1786177005.049829:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177005.049831:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177005.049832:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a300.
00000100:00000001:1.0:1786177005.049836:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177005.049838:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049842:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177005.049845:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177005.049847:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177005.049848:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177005.049854:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177001/real 1786177005]  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177005.049864:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177005.049866:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177005.049870:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8d1b8.
00000400:00000200:1.0:1786177005.049874:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177005.049876:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177005.049880:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177005.049884:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177005.049895:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049910:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177005.049911:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049914:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177005.049915:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177005.049919:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049927:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177005.049930:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.049938:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177005.049941:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049943:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177005.049945:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.049947:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177005.049948:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177005.049951:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177005.049956:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177005.049959:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177005.049963:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177005.049973:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856324736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177005.049978:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177005.049982:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177001/real 1786177005]  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177005.049991:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177005.049993:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177005.049996:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177005.049999:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177005.050001:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177005.050003:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177005.050007:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177005.050016:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.050022:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177005.050023:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.050025:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177005.050027:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177005.050031:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.050038:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177005.050043:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.050051:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177005.050053:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.050055:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177005.050057:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.050058:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177005.050060:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177005.050063:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177005.050067:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177005.050069:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177005.050074:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177005.050083:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856324864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177005.050089:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.050091:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177005.050095:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a680 x1872934856324736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.050103:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177005.050105:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177005.050107:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177005.050110:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177005.050112:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177005.050113:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177005.050115:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177005.050117:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:1.0:1786177005.050120:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177005.050121:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177005.050123:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a680.
00000100:00000001:1.0:1786177005.050128:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177005.050129:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.050131:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177005.050134:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419500 x1872934856324864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177026 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177005.050142:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177005.050143:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177005.050145:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177005.050149:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177005.050150:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177005.050151:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177005.050153:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177005.050155:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
02000000:00000001:1.0:1786177005.050157:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177005.050158:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177005.050160:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419500.
00000100:00000001:1.0:1786177005.050178:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177005.050180:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.050181:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177005.050191:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177005.050194:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177005.050196:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177005.050198:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177005.624601:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177005.624608:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177005.624627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177005.624628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177005.624631:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177005.624632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177005.624699:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177005.624705:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177005.624710:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177005.816645:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301901518, at 4301901568
00000100:00000040:1.0:1786177005.816647:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301901518, at 4301901568
00000100:00000010:1.0:1786177005.816666:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1e40.
00000100:00000010:0.0:1786177005.816674:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1420.
00000100:00000001:3.0:1786177006.072630:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177006.072641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177006.072649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177006.072658:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177006.072663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177006.072669:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177006.072673:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177006.072677:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177006.072680:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177006.648642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177006.648651:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177006.648656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177006.648657:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177006.648659:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177006.648660:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177006.648750:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177006.648755:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177006.648759:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177007.096658:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177007.096667:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786177007.096668:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177007.096670:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177007.096678:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:0.0:1786177007.096683:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177007.096684:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177007.096691:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177007.096692:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177007.096699:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1786177007.096700:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177007.096705:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177007.096705:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177007.096715:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177007.096721:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7230
00000400:00000200:0.0:1786177007.096730:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177007.096736:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177007.096738:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177007.096741:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.096744:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177007.096746:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.096748:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177007.096751:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177007.096752:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177007.096754:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177007.096757:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177007.096758:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177007.096760:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177007.096762:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.096763:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177007.096765:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.096766:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177007.096768:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177007.096770:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177007.096771:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177007.096774:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177007.096777:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177007.096779:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177007.096780:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177007.096782:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177007.096787:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41aa00.
00000020:00000040:0.0:1786177007.096790:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177007.096793:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177007.096795:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177007.096797:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177007.096799:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177007.096803:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177007.096809:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4000.
00000100:00000001:0.0:1786177007.096814:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177007.096820:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177007.096829:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177007.096832:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177007.096836:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da0255a4] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177007.096860:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177007.096863:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177007.096865:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177007.096866:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177007.096867:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177007.096869:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177007.096871:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7230
00000400:00000200:0.0:1786177007.096874:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177007.096876:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177007.096886:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177007.096889:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.096891:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177007.096897:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.096900:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177007.096903:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177007.096905:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177007.096907:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177007.096909:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177007.096911:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177007.096914:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177007.096917:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177007.096918:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177007.096920:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177007.096925:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177007.096925:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177007.096930:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177007.096931:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1786177007.096934:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177007.096942:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:1.0:1786177007.096944:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177007.096945:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177007.096950:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177007.096956:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177007.096959:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1786177007.096961:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.096963:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177007.096965:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786177007.096969:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:0.0:1786177007.096971:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786177007.096974:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:0.0:1786177007.096978:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.096980:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177007.096983:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177007.096988:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856324992:192.168.202.110@tcp:250:kworker.0
00000100:00000010:0.0:1786177007.096990:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41ad80.
00000100:00000001:1.0:1786177007.096996:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:0.0:1786177007.096997:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177007.097001:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177007.097002:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.097003:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1786177007.097005:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177007.097008:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177007.097008:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177007.097012:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.097014:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177007.097019:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
02000000:00000010:1.0:1786177007.097023:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b02c00.
00000100:00000001:0.0:1786177007.097025:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.097027:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177007.097035:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177007.097041:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00100000:0.0:1786177007.097049:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786177007.097051:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000001:0.0:1786177007.097062:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1786177007.097063:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856324992, portal 25
00000100:00000001:0.0:1786177007.097065:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.097069:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177007.097073:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761665536 : -114547947886080 : ffff97d1ba41aa00)
00000100:00000040:0.0:1786177007.097074:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034e2c676] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786177007.097086:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177007.097089:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177007.097095:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786177007.097102:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:0.0:1786177007.097103:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177007.097108:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:1.0:1786177007.097109:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177007.097111:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1786177007.097114:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000001:0.0:1786177007.097117:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:1.0:1786177007.097120:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856324992, offset 0
00000100:00080000:0.0:1786177007.097124:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7230
00000400:00000200:1.0:1786177007.097134:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177007.097138:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177007.097145:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177007.097151:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786177007.097151:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177007.097158:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.097160:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786177007.097164:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:0.0:1786177007.097167:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.097171:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177007.097174:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:1.0:1786177007.097174:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:0.0:1786177007.097178:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177007.097180:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786177007.097183:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:0.0:1786177007.097185:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177007.097189:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786177007.097190:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:0.0:1786177007.097194:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786177007.097196:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:0.0:1786177007.097204:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177007.097206:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254710:254710:256:4294967295] 192.168.202.110@tcp LPNI seq info [254699:254699:8:4294967295]
00000100:00000001:0.0:1786177007.097210:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177007.097218:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177007.097218:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177007.097224:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177007.097227:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786177007.097233:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177007.097238:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177007.097240:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:0.0:1786177007.097242:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177007.097247:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177007.097252:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786177007.097253:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177007.097256:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177007.097260:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000800:00000010:1.0:1786177007.097262:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000008:00000001:0.0:1786177007.097263:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177007.097268:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177007.097274:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b480.
00000800:00000200:1.0:1786177007.097275:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:0.0:1786177007.097279:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000800:00000200:1.0:1786177007.097282:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
02000000:00000001:0.0:1786177007.097284:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177007.097287:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177007.097291:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:1.1:1786177007.097294:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:0.0:1786177007.097295:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177007.097298:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786177007.097302:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
02000000:00000010:0.0:1786177007.097305:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000800:00000100:1.0:1786177007.097311:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177007.097313:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786177007.097316:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000100:00100000:0.0:1786177007.097323:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177007.097324:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177007.097335:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1786177007.097342:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786177007.097344:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000100:00000001:0.0:1786177007.097346:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177007.097350:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177007.097357:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:0.0:1786177007.097361:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001215e9e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177007.097366:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177007.097393:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.097396:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1786177007.097399:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1786177007.097402:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177007.097406:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177007.097411:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177007.097414:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177007.097417:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177007.097420:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786177007.097421:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:0.0:1786177007.097428:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7230
00000100:00000001:1.0:1786177007.097430:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177007.097435:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177007.097439:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177007.097444:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786177007.097444:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301906637
00000100:00080000:0.0:1786177007.097450:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000010:1.0:1786177007.097452:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1e40.
00000100:00000040:0.0:1786177007.097455:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.097459:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177007.097462:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:0.0:1786177007.097463:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.097468:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177007.097471:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786177007.097473:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301906637
00000020:00000040:0.0:1786177007.097474:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177007.097479:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000010:1.0:1786177007.097481:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1900.
00000020:00000040:0.0:1786177007.097483:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:1.0:1786177007.097487:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000020:00000001:0.0:1786177007.097488:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177007.097493:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:1.0:1786177007.097494:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1786177007.097498:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177007.097500:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177007.097504:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177007.097506:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:0.0:1786177007.097541:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.097545:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177007.097549:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177007.097554:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177007.097557:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177007.097560:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177007.097563:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177007.097566:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177007.097569:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177007.097574:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177007.097578:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177007.097580:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177007.097585:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177007/real 1786177007]  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1786177007.097587:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b100.
00000020:00000040:0.0:1786177007.097594:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177007.097610:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177007.097613:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786177007.097614:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1786177007.097616:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000001:1.0:1786177007.097618:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:0.0:1786177007.097620:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:1.0:1786177007.097624:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
02000000:00000001:0.0:1786177007.097625:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177007.097632:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
02000000:00000010:0.0:1786177007.097635:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000400:00000010:1.0:1786177007.097636:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177007.097643:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1786177007.097644:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177007.097649:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177007.097655:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1786177007.097670:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177007.097681:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177007.097683:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177007.097688:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000fa632fb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177007.097715:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.097720:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00080000:0.0:1786177007.097721:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786177007.097724:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177007.097726:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1786177007.097729:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177007.097730:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177007.097734:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177007.097734:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177007.097740:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:1.0:1786177007.097744:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177007.097746:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7230
00000400:00000200:0.0:1786177007.097756:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177007.097761:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1786177007.097764:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177007.097768:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177007.097771:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177007.097775:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.097782:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177007.097787:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177007.097792:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177007.097793:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177007.097799:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786177007.097799:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1786177007.097802:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:1.0:1786177007.097804:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000001:0.0:1786177007.097807:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786177007.097808:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.097813:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000040:0.0:1786177007.097814:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:1.0:1786177007.097816:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:0.0:1786177007.097819:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00080000:1.0:1786177007.097822:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000020:00000001:0.0:1786177007.097824:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00080000:1.0:1786177007.097832:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000040:0.0:1786177007.097833:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177007.097838:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177007.097838:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177007.097844:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:1.0:1786177007.097847:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177007.097849:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177007.097856:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177007.097864:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:1.0:1786177007.097867:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856324992:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177007.097870:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177007.097875:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786177007.097876:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177007.097879:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177007.097880:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1786177007.097885:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177007.097886:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41aa00 x1872934856324992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177007.097890:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177007.097895:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786177007.097900:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000008:00000001:0.0:1786177007.097901:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.097905:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:0.0:1786177007.097906:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177007.097910:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b02c00.
00000100:00000010:0.0:1786177007.097913:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419180.
02000000:00000001:1.0:1786177007.097916:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000040:0.0:1786177007.097918:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177007.097920:0:2391:0:(genops.c:1238:class_import_put()) Process entered
02000000:00000001:0.0:1786177007.097923:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000020:00000040:1.0:1786177007.097924:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177007.097927:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1786177007.097929:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00000001:0.0:1786177007.097932:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:1.0:1786177007.097934:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177007.097936:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177007.097940:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.097940:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177007.097945:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
02000000:00000010:0.0:1786177007.097948:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000100:00000010:1.0:1786177007.097949:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41aa00.
00000100:00000001:1.0:1786177007.097956:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177007.097957:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.097958:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.097963:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177007.097968:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177007.097973:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177007.097977:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177007.097979:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177007.097982:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177007.097984:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.097985:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177007.097989:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177007.097992:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005a89d8ab] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786177007.097998:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177007.098016:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.098018:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177007.098022:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177007.098026:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7240)
02000000:00000001:1.0:1786177007.098026:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177007.098030:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177007.098035:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177007.098039:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856325120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177007.098043:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177007.098045:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177007.098047:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.098050:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177007.098055:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177007.098063:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177007.098068:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177007.098072:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177007.098076:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177007.098080:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856325120, portal 10
00000100:00000001:1.0:1786177007.098083:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177007.098084:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761666432 : -114547947885184 : ffff97d1ba41ad80)
00000100:00000040:1.0:1786177007.098092:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098100:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177007.098104:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177007.098107:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177007.098111:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856325120, offset 0
00000400:00000200:1.0:1786177007.098116:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177007.098121:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177007.098125:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177007.098129:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177007.098132:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177007.098135:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177007.098138:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177007.098141:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254711:254711:256:4294967295] 192.168.202.110@tcp LPNI seq info [254700:254700:8:4294967295]
00000400:00000200:1.0:1786177007.098145:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177007.098158:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177007.098164:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177007.098168:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177007.098173:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177007.098176:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786177007.098181:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177007.098184:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1786177007.098188:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177007.098190:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177007.098194:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177007.098198:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177007.098202:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177007.098204:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177007.098207:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177007.098211:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098226:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177007.098229:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098237:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.098239:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177007.098243:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177007.098247:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786177007.098251:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177007.098253:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301902858
00000100:00000001:1.0:1786177007.098255:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301906637
00000100:00000001:1.0:1786177007.098258:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301902858
00000100:00000001:1.0:1786177007.098260:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301906637
00000100:00000001:1.0:1786177007.098262:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177007.098264:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.098266:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177007.098270:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177007/real 1786177007]  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177007.098279:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177007.098280:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177007.098282:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177007.098285:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177007.098287:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177007.098289:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177007.098292:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177007.098301:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098308:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177007.098310:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098312:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177007.098314:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177007.098318:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098325:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177007.098329:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098336:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177007.098338:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098340:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177007.098342:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098344:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177007.098346:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177007.098349:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177007.098352:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177007.098354:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177007.098359:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177007.098368:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856325120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177007.098372:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177007.098376:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098383:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177007.098384:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.098386:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177007.098388:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177007.098390:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177007.098393:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856325248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177007.098395:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177007.098397:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177007.098398:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.098399:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177007.098402:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177007.098404:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177007.098406:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177007.098408:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177007.098410:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177007.098412:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856325248, portal 10
00000100:00000001:1.0:1786177007.098414:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177007.098415:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761668224 : -114547947883392 : ffff97d1ba41b480)
00000100:00000040:1.0:1786177007.098419:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098423:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177007.098425:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177007.098426:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177007.098427:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856325248, offset 0
00000400:00000200:1.0:1786177007.098430:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177007.098433:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177007.098435:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177007.098437:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177007.098439:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177007.098440:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177007.098442:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177007.098444:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254712:254712:256:4294967295] 192.168.202.110@tcp LPNI seq info [254701:254701:8:4294967295]
00000400:00000200:1.0:1786177007.098447:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177007.098451:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177007.098454:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177007.098456:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177007.098459:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177007.098461:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786177007.098464:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177007.098466:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1786177007.098469:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177007.098470:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177007.098472:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177007.098475:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177007.098476:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177007.098478:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177007.098479:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177007.098481:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098487:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177007.098489:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098493:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.098494:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177007.098496:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177007.098498:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177007.098500:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177007.098502:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301902983
00000100:00000001:1.0:1786177007.098503:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301906637
00000100:00000001:1.0:1786177007.098505:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301902983
00000100:00000001:1.0:1786177007.098506:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301906637
00000100:00000001:1.0:1786177007.098508:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177007.098528:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.098530:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177007.098533:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177007/real 1786177007]  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177007.098538:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177007.098540:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177007.098541:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177007.098542:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177007.098544:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177007.098545:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177007.098547:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177007.098553:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098557:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177007.098558:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098559:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177007.098560:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177007.098562:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098566:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177007.098568:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098572:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177007.098574:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098575:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177007.098575:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098577:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177007.098578:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177007.098580:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177007.098582:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177007.098583:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177007.098586:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177007.098595:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856325248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177007.098613:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177007.098617:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098621:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177007.098622:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.098624:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177007.098625:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177007.098627:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177007.098629:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856325376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177007.098631:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177007.098632:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177007.098633:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.098635:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177007.098637:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177007.098638:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177007.098641:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177007.098643:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177007.098645:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177007.098647:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856325376, portal 4
00000100:00000001:1.0:1786177007.098649:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177007.098650:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761667328 : -114547947884288 : ffff97d1ba41b100)
00000100:00000040:1.0:1786177007.098654:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098658:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177007.098660:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177007.098661:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177007.098663:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856325376, offset 0
00000400:00000200:1.0:1786177007.098665:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177007.098669:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177007.098671:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177007.098673:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177007.098675:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177007.098677:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177007.098678:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177007.098680:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254713:254713:256:4294967295] 192.168.202.110@tcp LPNI seq info [254702:254702:8:4294967295]
00000400:00000200:1.0:1786177007.098683:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177007.098687:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177007.098690:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177007.098692:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177007.098695:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177007.098697:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786177007.098699:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177007.098701:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1786177007.098703:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177007.098705:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177007.098707:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177007.098709:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177007.098711:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177007.098712:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177007.098714:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177007.098716:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098721:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177007.098723:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098728:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.098729:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177007.098731:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177007.098733:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177007.098735:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177007.098737:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301902859
00000100:00000001:1.0:1786177007.098738:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301906637
00000100:00000001:1.0:1786177007.098740:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301902859
00000100:00000001:1.0:1786177007.098741:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301906637
00000100:00000001:1.0:1786177007.098742:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177007.098744:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.098745:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177007.098747:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177007/real 1786177007]  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177007.098751:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177007.098752:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177007.098753:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177007.098755:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177007.098756:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177007.098757:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177007.098759:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177007.098764:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098768:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177007.098769:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098770:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177007.098772:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177007.098775:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098782:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177007.098786:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098793:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177007.098796:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098798:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177007.098799:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098801:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177007.098803:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177007.098805:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177007.098809:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177007.098812:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177007.098816:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177007.098826:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856325376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177007.098830:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098832:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177007.098835:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41ad80 x1872934856325120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098842:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177007.098844:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177007.098846:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177007.098848:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177007.098849:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177007.098850:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177007.098852:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177007.098853:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:1.0:1786177007.098855:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177007.098856:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177007.098857:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41ad80.
00000100:00000001:1.0:1786177007.098860:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177007.098860:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098862:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177007.098864:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b480 x1872934856325248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098873:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177007.098873:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177007.098874:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177007.098876:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177007.098877:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177007.098877:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177007.098878:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177007.098879:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:1.0:1786177007.098881:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177007.098881:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177007.098882:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b480.
00000100:00000001:1.0:1786177007.098883:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177007.098884:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098885:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177007.098887:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b100 x1872934856325376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098890:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177007.098891:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177007.098892:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177007.098893:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177007.098894:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177007.098895:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177007.098896:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177007.098896:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:1.0:1786177007.098897:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177007.098898:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177007.098899:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b100.
00000100:00000001:1.0:1786177007.098900:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177007.098901:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098902:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.098905:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177007.098906:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.098907:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177007.098908:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177007.098909:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177007.098912:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098916:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177007.098917:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.098919:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177007.098920:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177007.098921:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177007.098923:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856325504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177007.098925:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177007.098927:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177007.098928:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177007.098929:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177007.098931:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177007.098933:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177007.098935:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177007.098937:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177007.098940:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177007.098943:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856325504, portal 4
00000100:00000001:1.0:1786177007.098945:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177007.098946:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761659264 : -114547947892352 : ffff97d1ba419180)
00000100:00000040:1.0:1786177007.098952:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.098959:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177007.098962:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177007.098964:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177007.098966:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856325504, offset 0
00000400:00000200:1.0:1786177007.098971:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177007.098974:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177007.098978:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177007.098981:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177007.098983:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177007.098986:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177007.098988:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177007.098992:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254714:254714:256:4294967295] 192.168.202.110@tcp LPNI seq info [254703:254703:8:4294967295]
00000400:00000200:1.0:1786177007.098997:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177007.099003:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177007.099008:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177007.099012:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177007.099016:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177007.099019:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1786177007.099024:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177007.099027:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1786177007.099030:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177007.099032:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177007.099036:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177007.099039:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177007.099042:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177007.099043:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177007.099046:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177007.099050:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.099057:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177007.099060:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177032 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.099066:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.099068:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177007.099071:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177007.099074:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177007.099078:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177007.099079:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301902867
00000100:00000001:1.0:1786177007.099082:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301906637
00000100:00000001:1.0:1786177007.099084:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301902867
00000100:00000001:1.0:1786177007.099087:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301906637
00000100:00000001:1.0:1786177007.099089:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177007.099092:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.099093:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177007.099098:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177007/real 1786177007]  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177007.099107:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177007.099108:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177007.099110:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177007.099112:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177007.099114:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177007.099116:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177007.099119:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177007.099126:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.099130:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177007.099131:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.099133:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177007.099134:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177007.099136:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.099140:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177007.099143:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.099147:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177007.099148:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.099149:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177007.099150:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.099152:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177007.099153:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177007.099155:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177007.099157:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177007.099158:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177007.099161:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177007.099166:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856325504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177007.099168:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.099169:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177007.099171:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419180 x1872934856325504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177032 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177007.099175:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177007.099176:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177007.099177:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177007.099178:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177007.099179:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177007.099181:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177007.099182:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177007.099183:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:1.0:1786177007.099185:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177007.099185:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177007.099186:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419180.
00000100:00000001:1.0:1786177007.099188:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177007.099189:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.099190:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177007.099192:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177007.099193:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177007.099195:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177007.099196:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177007.672670:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177007.672678:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177007.672683:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177007.672684:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177007.672687:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177007.672688:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177007.672765:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177007.672769:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177007.672773:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177008.120610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177008.120619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177008.120621:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177008.120628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177008.120631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177008.120635:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177008.120638:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177008.120640:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177008.120644:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177008.696647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177008.696653:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177008.696658:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177008.696658:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177008.696661:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177008.696662:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177008.696726:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177008.696729:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177008.696732:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177009.144627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177009.144633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177009.144634:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177009.144638:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177009.144641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177009.144644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177009.144648:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177009.144650:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177009.144652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177009.720673:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177009.720680:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177009.720686:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177009.720688:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177009.720711:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177009.720712:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177009.720817:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177009.720824:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177009.720829:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177010.168647:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177010.168654:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177010.168655:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177010.168660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177010.168663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177010.168665:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177010.168668:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177010.168670:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177010.168672:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177010.744633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177010.744640:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177010.744642:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177010.744644:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177010.744649:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177010.744650:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177010.744652:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177010.744654:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177010.744656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177010.744657:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177010.744659:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177010.744660:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177010.744661:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177010.744661:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177010.744662:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177010.744664:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177010.744666:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177010.744667:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177010.744724:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177010.744729:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177010.744733:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177010.936658:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301906637, at 4301906688
00000100:00000040:0.0:1786177010.936660:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301906637, at 4301906688
00000100:00000010:1.0:1786177010.936674:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1900.
00000100:00000010:0.0:1786177010.936680:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1e40.
00000100:00000001:3.0:1786177011.192662:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177011.192669:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177011.192672:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177011.192677:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177011.192680:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177011.192684:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177011.192686:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177011.192689:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177011.192692:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177011.768658:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177011.768666:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177011.768673:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177011.768675:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177011.768678:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177011.768680:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177011.768788:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177011.768793:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177011.768796:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177012.216659:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:0.0:1786177012.216664:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177012.216670:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177012.216674:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177012.216678:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177012.216684:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1786177012.216684:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:0.0:1786177012.216691:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177012.216692:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177012.216697:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177012.216700:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177012.216701:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177012.216706:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177012.216710:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177012.216715:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7235
00000400:00000200:0.0:1786177012.216730:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177012.216738:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177012.216742:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177012.216751:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177012.216756:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177012.216758:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177012.216761:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177012.216766:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177012.216768:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177012.216771:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177012.216774:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177012.216777:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177012.216780:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177012.216783:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177012.216785:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177012.216787:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177012.216789:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177012.216792:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177012.216796:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177012.216798:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177012.216802:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177012.216806:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177012.216809:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177012.216812:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177012.216814:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177012.216824:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418e00.
00000020:00000040:0.0:1786177012.216828:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177012.216832:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177012.216834:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177012.216837:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177012.216840:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177012.216845:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177012.216852:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000100:00000001:0.0:1786177012.216859:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177012.216869:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177012.216880:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177012.216885:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177012.216890:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003428892b] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177012.216914:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177012.216917:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177012.216920:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177012.216922:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177012.216925:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177012.216929:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177012.216934:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7235
00000400:00000200:0.0:1786177012.216938:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177012.216940:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177012.216947:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000400:00000200:0.0:1786177012.216948:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786177012.216951:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1786177012.216953:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177012.216963:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177012.216964:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177012.216976:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177012.216980:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177012.216982:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177012.216986:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177012.216989:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177012.216993:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177012.216994:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177012.216999:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177012.217002:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177012.217008:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00100000:1.0:1786177012.217008:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856325632:192.168.202.110@tcp:250:kworker.0
00000020:00000001:0.0:1786177012.217021:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786177012.217021:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:0.0:1786177012.217028:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
02000000:00000001:1.0:1786177012.217028:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177012.217033:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177012.217035:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:1.0:1786177012.217036:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:0.0:1786177012.217043:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000010:1.0:1786177012.217047:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177012.217053:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177012.217054:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177012.217061:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786177012.217065:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177012.217072:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:1.0:1786177012.217073:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:0.0:1786177012.217082:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000010:1.0:1786177012.217085:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000020:00000040:0.0:1786177012.217090:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000200:1.0:1786177012.217096:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856325632, portal 25
00000100:00080000:0.0:1786177012.217098:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177012.217101:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177012.217107:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761658368 : -114547947893248 : ffff97d1ba418e00)
00000100:00000001:0.0:1786177012.217108:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177012.217116:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1786177012.217116:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.217126:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1786177012.217127:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1786177012.217131:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177012.217136:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
02000000:00000001:0.0:1786177012.217138:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177012.217140:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856325632, offset 0
00000100:00000001:0.0:1786177012.217146:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:1.0:1786177012.217152:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000008:00100000:0.0:1786177012.217153:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177012.217159:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177012.217162:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177012.217166:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1786177012.217170:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418a80.
00000020:00000040:0.0:1786177012.217176:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000400:00000200:1.0:1786177012.217176:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:0.0:1786177012.217190:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:1.0:1786177012.217191:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1786177012.217194:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177012.217198:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1786177012.217206:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1786177012.217211:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1786177012.217212:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177012.217218:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177012.217221:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000010:0.0:1786177012.217227:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000400:00000200:1.0:1786177012.217231:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254715:254715:256:4294967295] 192.168.202.110@tcp LPNI seq info [254704:254704:8:4294967295]
00000100:00000001:0.0:1786177012.217236:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177012.217243:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:0.0:1786177012.217249:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177012.217268:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177012.217279:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000200:1.0:1786177012.217284:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177012.217285:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786177012.217294:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000100:00000040:0.0:1786177012.217299:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000606c34cb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1786177012.217309:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177012.217317:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786177012.217333:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177012.217336:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786177012.217338:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00080000:0.0:1786177012.217345:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000100:1.0:1786177012.217348:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177012.217353:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000100:00080000:0.0:1786177012.217355:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177012.217360:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1786177012.217361:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:0.0:1786177012.217363:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786177012.217371:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1786177012.217374:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786177012.217377:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177012.217381:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00080000:0.0:1786177012.217382:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7235
00000100:00000001:1.0:1786177012.217387:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:0.0:1786177012.217392:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:1.0:1786177012.217394:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177012.217406:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786177012.217409:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1786177012.217411:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177012.217415:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177012.217424:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177012.217427:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.217431:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177012.217433:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177012.217439:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177012.217441:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177012.217448:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177012.217450:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177012.217452:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301911757
00000020:00000001:0.0:1786177012.217459:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000010:1.0:1786177012.217459:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1900.
00000020:00000040:0.0:1786177012.217468:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177012.217473:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786177012.217474:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000040:0.0:1786177012.217478:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000001:1.0:1786177012.217482:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301911757
00000020:00000001:0.0:1786177012.217485:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000010:1.0:1786177012.217487:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1600.
00000100:00000001:1.0:1786177012.217492:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000020:00000001:0.0:1786177012.217496:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:1.0:1786177012.217497:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177012.217502:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.217507:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177012.217544:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177012/real 1786177012]  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177012.217562:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177012.217576:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:1.0:1786177012.217578:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177012.217581:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:0.0:1786177012.217586:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:1.0:1786177012.217586:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177012.217594:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000100:00000001:0.0:1786177012.217596:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000010:1.0:1786177012.217598:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177012.217603:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:0.0:1786177012.217606:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000200:1.0:1786177012.217611:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177012.217619:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:1.0:1786177012.217629:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177012.217630:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177012.217640:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177012.217650:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177012.217653:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:0.0:1786177012.217656:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.217658:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177012.217661:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177012.217664:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000008:00100000:0.0:1786177012.217666:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1786177012.217669:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:0.0:1786177012.217675:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177012.217681:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177012.217698:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.217700:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177012.217706:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1786177012.217708:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419180.
00000020:00000040:0.0:1786177012.217723:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000001:1.0:1786177012.217727:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177012.217732:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177012.217734:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786177012.217736:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177012.217740:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177012.217741:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.217746:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177012.217748:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786177012.217749:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177012.217757:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:0.0:1786177012.217758:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00080000:1.0:1786177012.217764:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
02000000:00000001:0.0:1786177012.217765:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.217770:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177012.217775:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786177012.217776:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:0.0:1786177012.217791:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177012.217793:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856325632:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177012.217801:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177012.217803:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177012.217807:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177012.217818:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418e00 x1872934856325632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177012.217827:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177012.217835:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.217837:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177012.217845:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177012.217848:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177012.217856:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000040:0.0:1786177012.217857:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005a89d8ab] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1786177012.217858:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177012.217864:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786177012.217869:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177012.217873:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:1.0:1786177012.217883:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177012.217886:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177012.217889:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418e00.
00000100:00000001:0.0:1786177012.217891:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.217896:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00080000:0.0:1786177012.217897:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786177012.217898:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.217903:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177012.217912:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177012.217914:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786177012.217916:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786177012.217917:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177012.217919:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.217923:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177012.217929:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177012.217931:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786177012.217931:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177012.217937:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177012.217942:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7235
00000100:00000001:1.0:1786177012.217964:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000400:00000200:0.0:1786177012.217965:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177012.217967:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177012.217974:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177012.217976:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1786177012.217980:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:1.0:1786177012.217982:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177012.217984:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00100000:1.0:1786177012.217991:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856325760:192.168.202.110@tcp:38:kworker.0
00000100:00000040:0.0:1786177012.217996:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177012.218000:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1786177012.218007:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1786177012.218008:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177012.218010:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177012.218013:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:1.0:1786177012.218015:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1786177012.218016:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177012.218022:0:62275:0:(genops.c:813:class_conn2export()) Process entered
02000000:00000010:1.0:1786177012.218023:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
00000020:00000040:0.0:1786177012.218028:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
02000000:00000001:1.0:1786177012.218029:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177012.218036:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177012.218039:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000400:00000200:1.0:1786177012.218043:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:0.0:1786177012.218050:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000010:1.0:1786177012.218051:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000020:00000001:0.0:1786177012.218058:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000010:1.0:1786177012.218061:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000040:0.0:1786177012.218062:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177012.218068:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177012.218070:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177012.218072:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177012.218074:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177012.218076:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177012.218077:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177012.218078:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856325760, portal 10
02000000:00000001:0.0:1786177012.218079:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177012.218087:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177012.218088:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177012.218092:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761657472 : -114547947894144 : ffff97d1ba418a80)
02000000:00000001:0.0:1786177012.218094:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177012.218101:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177012.218104:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000040:1.0:1786177012.218107:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1786177012.218109:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177012.218116:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177012.218128:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b100.
00000100:00000001:1.0:1786177012.218129:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:0.0:1786177012.218138:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000040:1.0:1786177012.218138:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177012.218146:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
02000000:00000001:0.0:1786177012.218148:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177012.218157:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177012.218158:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856325760, offset 0
00000100:00000001:0.0:1786177012.218168:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177012.218172:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1786177012.218176:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786177012.218178:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177012.218183:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1786177012.218185:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000400:00000200:1.0:1786177012.218189:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1786177012.218190:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177012.218198:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:0.0:1786177012.218204:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177012.218206:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177012.218219:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1786177012.218226:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177012.218231:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177012.218231:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:0.0:1786177012.218245:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000fa632fb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786177012.218249:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254716:254716:256:4294967295] 192.168.202.110@tcp LPNI seq info [254705:254705:8:4294967295]
00000400:00000200:1.0:1786177012.218272:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177012.218273:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177012.218283:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177012.218299:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1786177012.218300:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177012.218304:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177012.218310:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:1.0:1786177012.218319:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177012.218323:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:1.0:1786177012.218326:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000100:00080000:0.0:1786177012.218327:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7235
00000400:00000200:0.0:1786177012.218340:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:1.0:1786177012.218341:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177012.218350:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000400:00000200:0.0:1786177012.218354:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177012.218362:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:1.1:1786177012.218368:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786177012.218374:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:1.1:1786177012.218376:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:0.0:1786177012.218382:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177012.218389:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000100:1.0:1786177012.218391:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177012.218394:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000010:1.0:1786177012.218394:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000020:00000001:0.0:1786177012.218404:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177012.218405:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000400:00000200:1.0:1786177012.218406:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:0.0:1786177012.218410:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177012.218414:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000400:00000200:1.0:1786177012.218416:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1786177012.218418:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:1.0:1786177012.218421:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000020:00000001:0.0:1786177012.218424:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000010:1.0:1786177012.218426:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177012.218436:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:0.0:1786177012.218437:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177012.218441:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1786177012.218446:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177012.218447:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177012.218456:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177012.218466:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177012.218479:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177012.218481:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177012.218484:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177012.218491:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1786177012.218491:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177012.218498:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177012.218504:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.218504:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.218532:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177012.218539:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177012.218541:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1786177012.218542:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177012.218567:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177012.218568:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000100:00000001:0.0:1786177012.218571:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177012.218577:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1786177012.218578:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b480.
00000100:00000001:1.0:1786177012.218580:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301907979
00000020:00000040:0.0:1786177012.218583:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:1.0:1786177012.218584:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301911757
00000100:00000001:1.0:1786177012.218592:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301907979
02000000:00000001:0.0:1786177012.218593:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177012.218598:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.218599:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301911757
00000100:00000001:0.0:1786177012.218610:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786177012.218610:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177012.218617:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786177012.218617:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.218624:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000001:0.0:1786177012.218625:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177012.218632:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4000.
00000100:00100000:1.0:1786177012.218632:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177012/real 1786177012]  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177012.218639:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177012.218650:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:1.0:1786177012.218669:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1786177012.218670:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000001:1.0:1786177012.218676:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1786177012.218677:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177012.218689:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000100:00000040:0.0:1786177012.218690:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001215e9e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786177012.218701:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177012.218708:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:0.0:1786177012.218715:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.218717:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:0.0:1786177012.218727:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7245)
00000100:00000200:1.0:1786177012.218728:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177012.218740:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.218746:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177012.218747:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.218748:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177012.218749:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177012.218751:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.218756:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177012.218761:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.218766:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177012.218767:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.218768:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177012.218769:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.218770:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177012.218772:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177012.218774:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177012.218777:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177012.218779:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177012.218781:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177012.218788:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856325760:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177012.218792:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177012.218796:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.218804:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177012.218806:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177012.218809:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177012.218811:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177012.218814:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177012.218817:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856325888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177012.218821:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177012.218824:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177012.218825:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177012.218828:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177012.218832:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177012.218834:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177012.218837:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177012.218840:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177012.218842:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177012.218845:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856325888, portal 10
00000100:00000001:1.0:1786177012.218846:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177012.218847:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761659264 : -114547947892352 : ffff97d1ba419180)
00000100:00000040:1.0:1786177012.218852:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.218857:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177012.218859:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177012.218861:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177012.218863:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856325888, offset 0
00000400:00000200:1.0:1786177012.218865:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177012.218869:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177012.218872:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177012.218874:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177012.218876:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177012.218878:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177012.218880:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177012.218882:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254717:254717:256:4294967295] 192.168.202.110@tcp LPNI seq info [254706:254706:8:4294967295]
00000400:00000200:1.0:1786177012.218885:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177012.218890:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177012.218893:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177012.218895:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177012.218898:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177012.218900:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786177012.218904:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177012.218906:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1786177012.218909:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177012.218910:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177012.218913:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177012.218915:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177012.218917:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177012.218919:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177012.218920:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177012.218923:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.218928:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177012.218930:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.218935:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.218937:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177012.218939:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177012.218941:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177012.218943:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177012.218945:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301908104
00000100:00000001:1.0:1786177012.218947:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301911757
00000100:00000001:1.0:1786177012.218948:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301908104
00000100:00000001:1.0:1786177012.218950:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301911757
00000100:00000001:1.0:1786177012.218951:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177012.218952:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.218953:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177012.218956:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177012/real 1786177012]  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177012.218960:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177012.218961:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177012.218963:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177012.218964:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177012.218965:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177012.218967:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177012.218969:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177012.218974:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.218977:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177012.218978:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.218979:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177012.218980:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177012.218982:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.218986:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177012.218989:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.218994:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177012.218996:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.218997:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177012.218998:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.218999:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177012.219000:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177012.219002:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177012.219004:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177012.219005:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177012.219007:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177012.219012:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856325888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177012.219014:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219016:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177012.219018:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418a80 x1872934856325760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219022:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177012.219023:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177012.219025:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177012.219026:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177012.219027:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177012.219028:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177012.219030:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177012.219031:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:1.0:1786177012.219033:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177012.219035:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177012.219036:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418a80.
00000100:00000001:1.0:1786177012.219038:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177012.219039:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219040:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177012.219042:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419180 x1872934856325888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219047:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177012.219048:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177012.219048:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177012.219050:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177012.219051:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177012.219052:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177012.219053:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177012.219053:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:1.0:1786177012.219055:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177012.219055:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177012.219056:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419180.
00000100:00000001:1.0:1786177012.219057:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177012.219058:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219059:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219062:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177012.219063:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.219065:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177012.219066:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177012.219067:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177012.219069:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219074:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177012.219074:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177012.219076:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177012.219077:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177012.219078:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177012.219080:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856326016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177012.219082:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177012.219084:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177012.219085:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177012.219086:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177012.219088:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177012.219090:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177012.219092:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177012.219094:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177012.219096:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177012.219098:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856326016, portal 4
00000100:00000001:1.0:1786177012.219099:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177012.219100:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761667328 : -114547947884288 : ffff97d1ba41b100)
00000100:00000040:1.0:1786177012.219105:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219109:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177012.219111:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177012.219112:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177012.219114:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856326016, offset 0
00000400:00000200:1.0:1786177012.219116:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177012.219118:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177012.219120:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177012.219122:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177012.219124:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177012.219125:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177012.219127:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177012.219129:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254718:254718:256:4294967295] 192.168.202.110@tcp LPNI seq info [254707:254707:8:4294967295]
00000400:00000200:1.0:1786177012.219132:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177012.219136:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177012.219139:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177012.219140:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177012.219143:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177012.219144:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786177012.219147:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177012.219149:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1786177012.219152:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177012.219153:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177012.219155:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177012.219158:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177012.219160:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177012.219161:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177012.219163:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177012.219165:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219170:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177012.219172:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219176:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.219177:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177012.219179:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177012.219181:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177012.219183:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177012.219184:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301907979
00000100:00000001:1.0:1786177012.219185:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301911757
00000100:00000001:1.0:1786177012.219187:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301907979
00000100:00000001:1.0:1786177012.219189:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301911757
00000100:00000001:1.0:1786177012.219190:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177012.219192:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.219193:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177012.219195:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177012/real 1786177012]  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177012.219200:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177012.219201:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177012.219202:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177012.219203:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177012.219204:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177012.219206:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177012.219210:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177012.219225:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219233:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177012.219234:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219236:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177012.219238:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177012.219242:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219250:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177012.219255:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219262:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177012.219264:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219265:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177012.219266:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219268:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177012.219269:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177012.219270:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177012.219273:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177012.219275:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177012.219278:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177012.219283:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856326016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177012.219286:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177012.219288:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219297:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177012.219298:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177012.219300:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177012.219301:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177012.219302:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177012.219304:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856326144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177012.219306:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177012.219307:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177012.219309:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177012.219310:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177012.219312:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177012.219314:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177012.219316:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177012.219318:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177012.219320:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177012.219322:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856326144, portal 4
00000100:00000001:1.0:1786177012.219324:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177012.219325:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761668224 : -114547947883392 : ffff97d1ba41b480)
00000100:00000040:1.0:1786177012.219329:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219333:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177012.219335:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177012.219336:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177012.219338:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856326144, offset 0
00000400:00000200:1.0:1786177012.219340:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177012.219343:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177012.219345:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177012.219347:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177012.219349:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177012.219350:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177012.219352:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177012.219355:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254719:254719:256:4294967295] 192.168.202.110@tcp LPNI seq info [254708:254708:8:4294967295]
00000400:00000200:1.0:1786177012.219357:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177012.219362:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177012.219365:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177012.219367:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177012.219370:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177012.219372:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1786177012.219375:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177012.219376:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1786177012.219380:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177012.219381:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177012.219383:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177012.219386:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177012.219388:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177012.219389:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177012.219391:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177012.219393:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219398:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177012.219400:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219405:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.219406:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177012.219407:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177012.219410:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177012.219412:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177012.219413:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301907987
00000100:00000001:1.0:1786177012.219415:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301911757
00000100:00000001:1.0:1786177012.219417:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301907987
00000100:00000001:1.0:1786177012.219418:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301911757
00000100:00000001:1.0:1786177012.219420:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177012.219422:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.219423:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177012.219425:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177012/real 1786177012]  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177012.219431:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177012.219432:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177012.219433:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177012.219434:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177012.219435:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177012.219437:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177012.219439:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177012.219444:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219448:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177012.219449:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219450:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177012.219451:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177012.219453:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219457:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177012.219459:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219463:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177012.219464:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219465:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177012.219466:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219467:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177012.219468:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177012.219469:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177012.219472:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177012.219473:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177012.219476:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177012.219481:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856326144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177012.219483:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219484:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177012.219486:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b100 x1872934856326016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219490:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177012.219491:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177012.219492:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177012.219493:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177012.219494:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177012.219495:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177012.219496:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177012.219497:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:1.0:1786177012.219498:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177012.219499:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177012.219500:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b100.
00000100:00000001:1.0:1786177012.219502:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177012.219503:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219504:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177012.219506:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b480 x1872934856326144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177012.219530:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177012.219531:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177012.219532:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177012.219533:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177012.219534:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177012.219535:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177012.219536:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177012.219538:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4000.
02000000:00000001:1.0:1786177012.219539:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177012.219540:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177012.219541:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b480.
00000100:00000001:1.0:1786177012.219543:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177012.219544:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219545:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177012.219547:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177012.219560:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177012.219562:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177012.219564:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177012.792673:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177012.792682:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177012.792689:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177012.792690:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177012.792694:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177012.792698:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177012.792701:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177012.792705:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177012.792709:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177012.792712:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177012.792714:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177012.792717:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177012.792803:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177012.792808:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177012.792813:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177013.240640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177013.240648:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177013.240650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177013.240656:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177013.240660:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177013.240664:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177013.240667:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177013.240670:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177013.240673:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177013.816654:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177013.816660:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177013.816666:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177013.816668:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177013.816671:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177013.816673:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177013.816750:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177013.816756:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177013.816761:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177014.264624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177014.264630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177014.264632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177014.264636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177014.264639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177014.264642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177014.264644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177014.264645:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177014.264647:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177014.840616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177014.840621:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177014.840626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177014.840627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177014.840630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177014.840631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177014.840692:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177014.840696:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177014.840700:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177015.288638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177015.288646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177015.288649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177015.288654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177015.288658:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177015.288661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177015.288664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177015.288667:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177015.288670:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177015.864682:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177015.864692:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177015.864699:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177015.864701:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177015.864705:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177015.864707:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177015.864809:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177015.864816:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177015.864822:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177016.056663:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301911757, at 4301911808
00000100:00000040:1.0:1786177016.056663:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301911757, at 4301911808
00000100:00000010:0.0:1786177016.056685:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1900.
00000100:00000010:1.0:1786177016.056688:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1600.
00000100:00000001:3.0:1786177016.312670:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177016.312680:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177016.312682:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177016.312688:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177016.312692:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177016.312696:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177016.312699:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177016.312702:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177016.312704:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177016.888625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177016.888631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177016.888635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177016.888637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177016.888640:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177016.888642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177016.888693:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177016.888696:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177016.888699:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177017.336627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177017.336635:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177017.336638:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177017.336643:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1786177017.336645:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177017.336651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1786177017.336653:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177017.336655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177017.336658:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177017.336659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177017.336664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177017.336665:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177017.336666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177017.336674:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177017.336683:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7240
00000400:00000200:2.0:1786177017.336695:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177017.336702:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177017.336706:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177017.336710:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.336713:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177017.336716:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.336719:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177017.336723:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177017.336725:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177017.336728:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177017.336731:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177017.336733:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177017.336737:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177017.336740:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.336742:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177017.336744:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.336747:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177017.336750:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177017.336753:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177017.336756:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177017.336759:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177017.336763:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177017.336765:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177017.336768:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177017.336770:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177017.336780:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786177017.336784:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177017.336788:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177017.336790:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177017.336793:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177017.336796:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177017.336800:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177017.336808:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786177017.336815:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177017.336825:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177017.336836:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177017.336841:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177017.336846:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177017.336870:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177017.336874:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177017.336877:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177017.336879:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177017.336881:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177017.336884:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177017.336887:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7240
00000400:00000200:2.0:1786177017.336891:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177017.336895:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177017.336898:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177017.336901:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.336903:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177017.336905:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.336907:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177017.336910:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177017.336912:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177017.336914:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177017.336917:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177017.336919:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177017.336922:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177017.336926:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.336928:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177017.336930:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.336932:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177017.336934:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1786177017.336935:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786177017.336939:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177017.336943:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1786177017.336943:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177017.336949:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177017.336952:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177017.336953:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177017.336960:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177017.336962:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1786177017.336964:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.336965:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177017.336971:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786177017.336972:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177017.336976:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:2.0:1786177017.336978:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786177017.336979:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177017.336985:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856326272:192.168.202.110@tcp:250:kworker.0
00000008:00000001:2.0:1786177017.336986:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.336992:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1786177017.336993:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177017.336997:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:2.0:1786177017.336999:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
02000000:00000001:1.0:1786177017.337001:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177017.337005:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:2.0:1786177017.337005:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177017.337010:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000010:1.0:1786177017.337011:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:2.0:1786177017.337013:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177017.337015:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177017.337018:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177017.337021:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1786177017.337023:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177017.337025:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177017.337028:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
02000000:00000010:2.0:1786177017.337033:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000400:00000010:1.0:1786177017.337035:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000001:2.0:1786177017.337039:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177017.337043:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856326272, portal 25
00000100:00000001:1.0:1786177017.337046:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177017.337048:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00100000:2.0:1786177017.337049:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1786177017.337058:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177017.337066:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177017.337070:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786177017.337071:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177017.337076:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177017.337079:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177017.337083:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856326272, offset 0
00000100:00000040:2.0:1786177017.337083:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786177017.337094:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177017.337101:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177017.337105:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786177017.337107:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786177017.337113:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:2.0:1786177017.337114:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177017.337119:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1786177017.337120:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1786177017.337122:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786177017.337126:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1786177017.337128:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786177017.337132:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1786177017.337135:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7240
00000400:00000200:1.0:1786177017.337138:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177017.337146:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254720:254720:256:4294967295] 192.168.202.110@tcp LPNI seq info [254709:254709:8:4294967295]
00000400:00000200:2.0:1786177017.337147:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177017.337152:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177017.337156:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177017.337160:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786177017.337169:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1786177017.337170:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.337175:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1786177017.337177:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177017.337181:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000010:1.0:1786177017.337183:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000100:00000001:2.0:1786177017.337187:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:1.0:1786177017.337193:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786177017.337193:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000200:1.0:1786177017.337197:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000040:2.0:1786177017.337198:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177017.337203:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:1.1:1786177017.337207:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1786177017.337208:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000800:00000200:1.1:1786177017.337210:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:2.0:1786177017.337214:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000100:1.0:1786177017.337217:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:2.0:1786177017.337221:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000010:1.0:1786177017.337222:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177017.337230:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1786177017.337230:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.337235:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786177017.337237:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:2.0:1786177017.337242:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177017.337243:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000100:00000001:2.0:1786177017.337246:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000010:1.0:1786177017.337247:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177017.337253:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:2.0:1786177017.337254:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000200:1.0:1786177017.337259:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786177017.337261:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177017.337267:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177017.337272:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786177017.337276:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1786177017.337277:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1786177017.337282:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177017.337283:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177017.337288:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177017.337292:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1786177017.337294:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.337298:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000008:00000001:2.0:1786177017.337298:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177017.337303:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.337304:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177017.337308:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000400:00000200:1.0:1786177017.337310:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177017.337315:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301916877
00000020:00000040:2.0:1786177017.337315:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177017.337322:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000010:1.0:1786177017.337323:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1600.
02000000:00000001:2.0:1786177017.337325:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177017.337329:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786177017.337331:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:2.0:1786177017.337332:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177017.337336:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.337338:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301916877
02000000:00000010:2.0:1786177017.337343:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000010:1.0:1786177017.337344:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b11e0.
00000100:00000001:1.0:1786177017.337350:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786177017.337351:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.337353:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177017.337356:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.337359:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177017.337361:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:1.0:1786177017.337365:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177017/real 1786177017]  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177017.337383:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000001:1.0:1786177017.337386:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786177017.337388:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786177017.337389:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177017.337392:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177017.337396:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177017.337399:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000040:2.0:1786177017.337399:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.337401:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177017.337407:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177017.337428:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177017.337429:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786177017.337434:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786177017.337438:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177017.337442:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177017.337442:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1786177017.337447:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177017.337447:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786177017.337451:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177017.337452:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:1.0:1786177017.337457:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177017.337459:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177017.337466:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7240
00000100:00000001:1.0:1786177017.337467:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1786177017.337475:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1786177017.337476:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.337485:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000400:00000200:2.0:1786177017.337485:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786177017.337489:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.337491:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:2.0:1786177017.337491:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786177017.337493:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.337497:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177017.337551:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177017.337554:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177017.337558:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177017.337560:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177017.337565:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177017.337565:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.337572:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:1.0:1786177017.337580:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856326272:192.168.202.110@tcp:250:kworker.0
00000100:00000040:2.0:1786177017.337580:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.337587:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177017.337588:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1786177017.337593:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786177017.337594:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000040:2.0:1786177017.337598:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000040:1.0:1786177017.337599:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856326272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177017.337603:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177017.337608:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:1.0:1786177017.337609:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177017.337614:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000020:00000001:2.0:1786177017.337614:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
02000000:00000010:1.0:1786177017.337617:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
00000020:00000001:2.0:1786177017.337620:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
02000000:00000001:1.0:1786177017.337622:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177017.337625:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177017.337628:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000040:2.0:1786177017.337629:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000020:00000001:1.0:1786177017.337631:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177017.337635:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786177017.337636:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1786177017.337639:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000040:2.0:1786177017.337641:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:1.0:1786177017.337642:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177017.337647:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:2.0:1786177017.337647:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177017.337652:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177017.337654:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786177017.337654:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:1.0:1786177017.337659:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177017.337663:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177017.337666:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177017.337667:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.337669:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177017.337672:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786177017.337673:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:2.0:1786177017.337678:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177017.337682:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177017.337684:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177017.337686:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786177017.337688:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000008:00000001:2.0:1786177017.337690:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177017.337693:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.337697:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000010:2.0:1786177017.337699:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000100:00000040:1.0:1786177017.337705:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177017.337705:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177017.337710:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177017.337713:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177017.337717:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177017.337720:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786177017.337724:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1786177017.337725:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.337728:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177017.337733:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
02000000:00000001:1.0:1786177017.337734:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177017.337739:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177017.337740:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177017.337745:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177017.337751:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:1.0:1786177017.337755:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856326400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177017.337766:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177017.337769:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1786177017.337772:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1786177017.337773:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177017.337778:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177017.337779:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177017.337785:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
00000100:00000040:2.0:1786177017.337787:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786177017.337789:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177017.337799:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177017.337805:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:2.0:1786177017.337807:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177017.337813:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00080000:2.0:1786177017.337813:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:1.0:1786177017.337819:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856326400, portal 10
00000100:00080000:2.0:1786177017.337820:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1786177017.337825:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177017.337826:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786177017.337830:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00080000:2.0:1786177017.337831:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177017.337838:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177017.337841:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7240
00000100:00000040:1.0:1786177017.337845:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786177017.337849:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177017.337857:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786177017.337863:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1786177017.337864:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177017.337870:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177017.337874:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:1.0:1786177017.337876:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000001:2.0:1786177017.337881:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1786177017.337882:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856326400, offset 0
00000100:00000040:2.0:1786177017.337887:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177017.337889:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177017.337894:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786177017.337898:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1786177017.337900:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177017.337903:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000400:00000200:1.0:1786177017.337904:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:2.0:1786177017.337908:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786177017.337910:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:2.0:1786177017.337913:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000400:00000200:1.0:1786177017.337914:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177017.337919:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:2.0:1786177017.337920:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:1.0:1786177017.337922:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000001:2.0:1786177017.337925:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:1.0:1786177017.337927:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254721:254721:256:4294967295] 192.168.202.110@tcp LPNI seq info [254710:254710:8:4294967295]
00000400:00000200:1.0:1786177017.337935:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786177017.337935:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.337941:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786177017.337945:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1786177017.337948:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177017.337953:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:1.0:1786177017.337954:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177017.337959:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000020:00000040:2.0:1786177017.337960:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000800:00000200:1.0:1786177017.337966:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786177017.337969:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:1.0:1786177017.337970:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1786177017.337977:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1786177017.337981:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1786177017.337983:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:1.1:1786177017.337985:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
02000000:00000001:2.0:1786177017.337988:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000100:1.0:1786177017.337989:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177017.337995:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
02000000:00000001:2.0:1786177017.337995:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177017.338000:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1786177017.338001:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177017.338005:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:1.0:1786177017.338007:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000008:00000001:2.0:1786177017.338012:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177017.338014:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177017.338017:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:2.0:1786177017.338017:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.338022:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000010:2.0:1786177017.338025:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000100:00000200:1.0:1786177017.338027:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177017.338031:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177017.338038:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786177017.338039:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1786177017.338042:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177017.338044:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177017.338046:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177017.338049:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786177017.338054:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177017.338054:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.338060:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177017.338063:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177017.338063:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000040:1.0:1786177017.338067:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000100:00000001:2.0:1786177017.338069:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177017.338071:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177017.338075:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301913098
00000100:00000001:1.0:1786177017.338078:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301916877
00000100:00100000:2.0:1786177017.338080:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177017.338081:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301913098
00000100:00000001:1.0:1786177017.338085:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301916877
00000100:00000001:1.0:1786177017.338089:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177017.338094:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.338098:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1786177017.338102:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00100000:1.0:1786177017.338104:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177017/real 1786177017]  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177017.338107:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786177017.338116:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:2.0:1786177017.338117:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177017.338118:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177017.338122:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177017.338125:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177017.338128:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177017.338132:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1786177017.338137:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177017.338138:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177017.338145:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7250)
00000100:00000200:1.0:1786177017.338148:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338154:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177017.338155:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338156:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177017.338158:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177017.338160:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338165:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177017.338169:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338175:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177017.338177:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338178:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177017.338179:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338180:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177017.338181:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177017.338183:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177017.338185:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177017.338186:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177017.338188:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177017.338193:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856326400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177017.338196:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177017.338198:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338202:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177017.338203:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177017.338205:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177017.338206:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177017.338208:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177017.338209:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856326528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177017.338211:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177017.338212:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177017.338214:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177017.338215:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177017.338217:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177017.338218:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177017.338221:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177017.338222:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177017.338225:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177017.338226:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856326528, portal 10
00000100:00000001:1.0:1786177017.338228:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177017.338229:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187904 : -114548255363712 : ffff97d1a7eded80)
00000100:00000040:1.0:1786177017.338233:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338237:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177017.338239:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177017.338240:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177017.338241:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856326528, offset 0
00000400:00000200:1.0:1786177017.338244:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177017.338246:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177017.338248:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177017.338250:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177017.338251:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177017.338253:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177017.338254:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177017.338257:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254722:254722:256:4294967295] 192.168.202.110@tcp LPNI seq info [254711:254711:8:4294967295]
00000400:00000200:1.0:1786177017.338260:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177017.338264:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177017.338267:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177017.338269:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177017.338271:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177017.338273:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786177017.338275:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177017.338277:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1786177017.338279:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177017.338281:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177017.338282:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177017.338285:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177017.338287:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177017.338288:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177017.338290:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177017.338292:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338297:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177017.338299:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338302:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.338304:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177017.338305:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177017.338307:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177017.338309:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177017.338311:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301913223
00000100:00000001:1.0:1786177017.338312:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301916877
00000100:00000001:1.0:1786177017.338314:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301913223
00000100:00000001:1.0:1786177017.338315:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301916877
00000100:00000001:1.0:1786177017.338316:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177017.338317:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.338318:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177017.338320:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177017/real 1786177017]  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177017.338325:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177017.338326:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177017.338327:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177017.338328:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177017.338329:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177017.338331:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177017.338333:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177017.338338:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338343:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177017.338344:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338346:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177017.338347:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177017.338351:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338358:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177017.338362:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338369:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177017.338371:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338373:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177017.338374:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338376:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177017.338378:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177017.338381:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177017.338384:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177017.338387:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177017.338392:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177017.338401:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856326528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177017.338405:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338407:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177017.338410:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856326400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338415:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177017.338417:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177017.338418:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177017.338420:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177017.338421:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177017.338422:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177017.338424:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177017.338425:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:1.0:1786177017.338427:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177017.338428:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177017.338429:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:1.0:1786177017.338432:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177017.338432:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338434:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177017.338436:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eded80 x1872934856326528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338440:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177017.338441:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177017.338442:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177017.338444:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177017.338444:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177017.338445:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177017.338446:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177017.338447:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3400.
02000000:00000001:1.0:1786177017.338448:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177017.338449:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177017.338450:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eded80.
00000100:00000001:1.0:1786177017.338451:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177017.338452:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338453:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338455:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177017.338456:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.338457:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177017.338458:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177017.338459:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177017.338461:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338466:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177017.338467:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177017.338469:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177017.338470:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177017.338471:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177017.338473:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856326656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177017.338475:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177017.338476:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177017.338477:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177017.338478:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177017.338481:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177017.338482:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177017.338485:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177017.338487:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177017.338489:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177017.338491:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856326656, portal 4
00000100:00000001:1.0:1786177017.338493:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177017.338494:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454180736 : -114548255370880 : ffff97d1a7edd180)
00000100:00000040:1.0:1786177017.338498:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338533:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177017.338534:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177017.338537:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177017.338539:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856326656, offset 0
00000400:00000200:1.0:1786177017.338541:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177017.338544:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177017.338547:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177017.338549:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177017.338551:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177017.338553:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177017.338554:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177017.338557:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254723:254723:256:4294967295] 192.168.202.110@tcp LPNI seq info [254712:254712:8:4294967295]
00000400:00000200:1.0:1786177017.338560:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177017.338564:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177017.338568:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177017.338570:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177017.338572:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177017.338574:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786177017.338577:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177017.338579:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1786177017.338581:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177017.338582:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177017.338585:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177017.338587:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177017.338589:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177017.338590:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177017.338592:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177017.338594:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338600:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177017.338602:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338606:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.338607:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177017.338609:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177017.338612:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177017.338614:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177017.338615:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301913099
00000100:00000001:1.0:1786177017.338617:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301916877
00000100:00000001:1.0:1786177017.338619:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301913099
00000100:00000001:1.0:1786177017.338620:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301916877
00000100:00000001:1.0:1786177017.338622:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177017.338623:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.338624:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177017.338627:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177017/real 1786177017]  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177017.338632:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177017.338633:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177017.338634:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177017.338636:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177017.338637:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177017.338638:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177017.338640:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177017.338645:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338649:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177017.338650:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338651:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177017.338652:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177017.338654:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338658:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177017.338661:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338665:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177017.338666:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338667:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177017.338668:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338669:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177017.338670:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177017.338671:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177017.338673:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177017.338674:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177017.338677:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177017.338682:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856326656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177017.338685:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177017.338687:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338691:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177017.338692:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177017.338693:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177017.338694:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177017.338695:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177017.338697:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856326784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177017.338699:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177017.338700:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177017.338701:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177017.338702:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177017.338704:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177017.338705:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177017.338707:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177017.338708:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177017.338710:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177017.338712:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856326784, portal 4
00000100:00000001:1.0:1786177017.338713:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177017.338714:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178048 : -114548255373568 : ffff97d1a7edc700)
00000100:00000040:1.0:1786177017.338718:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338723:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177017.338724:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177017.338725:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177017.338727:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856326784, offset 0
00000400:00000200:1.0:1786177017.338730:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177017.338732:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177017.338734:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177017.338736:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177017.338737:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177017.338739:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177017.338740:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177017.338742:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254724:254724:256:4294967295] 192.168.202.110@tcp LPNI seq info [254713:254713:8:4294967295]
00000400:00000200:1.0:1786177017.338745:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177017.338749:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177017.338751:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177017.338753:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177017.338755:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177017.338757:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1786177017.338759:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177017.338761:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1786177017.338763:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177017.338765:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177017.338767:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177017.338769:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177017.338770:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177017.338771:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177017.338773:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177017.338775:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338780:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177017.338782:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338786:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.338787:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177017.338788:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177017.338790:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177017.338792:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177017.338793:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301913107
00000100:00000001:1.0:1786177017.338795:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301916877
00000100:00000001:1.0:1786177017.338796:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301913107
00000100:00000001:1.0:1786177017.338798:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301916877
00000100:00000001:1.0:1786177017.338799:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177017.338800:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.338801:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177017.338803:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177017/real 1786177017]  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177017.338807:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177017.338808:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177017.338809:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177017.338810:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177017.338812:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177017.338813:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177017.338815:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177017.338820:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338824:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177017.338825:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338826:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177017.338827:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177017.338829:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338833:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177017.338835:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338839:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177017.338840:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338841:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177017.338842:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338843:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177017.338844:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177017.338846:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177017.338848:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177017.338849:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177017.338851:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177017.338856:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856326784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177017.338858:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338859:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177017.338861:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd180 x1872934856326656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338865:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177017.338866:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177017.338867:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177017.338868:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177017.338869:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177017.338870:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177017.338871:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177017.338872:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:1.0:1786177017.338873:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177017.338874:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177017.338875:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd180.
00000100:00000001:1.0:1786177017.338877:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177017.338878:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338879:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177017.338881:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc700 x1872934856326784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177017.338889:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177017.338890:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177017.338891:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177017.338892:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177017.338893:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177017.338893:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177017.338894:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177017.338895:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:1.0:1786177017.338896:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177017.338897:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177017.338898:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc700.
00000100:00000001:1.0:1786177017.338899:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177017.338900:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338901:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177017.338903:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177017.338904:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177017.338905:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177017.338906:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177017.912616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177017.912622:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177017.912626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177017.912627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177017.912630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177017.912631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177017.912684:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177017.912688:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177017.912691:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177018.360595:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177018.360601:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177018.360603:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177018.360608:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177018.360611:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177018.360614:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177018.360617:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177018.360619:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177018.360622:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177018.936635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177018.936642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177018.936648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177018.936649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177018.936653:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177018.936654:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177018.936711:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177018.936714:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177018.936717:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177019.384611:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177019.384619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177019.384622:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177019.384628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177019.384632:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177019.384637:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177019.384640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177019.384642:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177019.384645:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177019.960602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177019.960608:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177019.960612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177019.960613:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177019.960615:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177019.960616:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177019.960670:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177019.960673:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177019.960676:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177020.408629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177020.408637:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177020.408639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177020.408645:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177020.408650:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177020.408654:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177020.408657:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177020.408660:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177020.408663:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177020.984593:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177020.984602:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177020.984605:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177020.984609:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177020.984613:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177020.984615:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177020.984618:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177020.984621:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177020.984625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177020.984627:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177020.984628:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177020.984630:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177020.984632:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177020.984633:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177020.984635:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177020.984637:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177020.984640:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177020.984642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177020.984694:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177020.984698:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177020.984701:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177021.176593:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301916877, at 4301916928
00000100:00000040:1.0:1786177021.176593:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301916877, at 4301916928
00000100:00000010:0.0:1786177021.176607:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1600.
00000100:00000010:1.0:1786177021.176613:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b11e0.
00000100:00000001:3.0:1786177021.432600:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177021.432608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177021.432610:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177021.432616:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177021.432620:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177021.432624:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177021.432626:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177021.432630:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177021.432633:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177022.008625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177022.008631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177022.008635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177022.008636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177022.008638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177022.008639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177022.008704:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177022.008709:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177022.008741:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786177022.456624:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177022.456632:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177022.456634:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177022.456637:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177022.456641:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177022.456647:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7245
00000400:00000200:2.0:1786177022.456655:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177022.456660:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177022.456671:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177022.456674:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.456677:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177022.456680:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.456682:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177022.456686:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177022.456687:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177022.456690:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177022.456693:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177022.456694:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177022.456696:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177022.456698:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.456699:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177022.456701:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.456702:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177022.456704:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177022.456707:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177022.456708:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177022.456711:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177022.456714:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177022.456715:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177022.456717:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177022.456718:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177022.456733:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786177022.456736:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177022.456739:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177022.456740:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177022.456743:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177022.456746:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177022.456749:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177022.456755:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786177022.456760:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177022.456770:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177022.456781:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177022.456786:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177022.456791:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177022.456836:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177022.456839:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177022.456841:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177022.456842:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177022.456844:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177022.456846:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177022.456849:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7245
00000400:00000200:2.0:1786177022.456853:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177022.456857:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177022.456859:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786177022.456862:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1786177022.456862:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.456867:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177022.456869:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1786177022.456869:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.456873:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177022.456874:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1786177022.456875:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000040:1.0:1786177022.456886:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177022.456893:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177022.456896:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177022.456898:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177022.456900:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177022.456901:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177022.456904:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.456906:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177022.456910:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177022.456913:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177022.456913:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177022.456918:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786177022.456919:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177022.456921:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:1.0:1786177022.456926:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1786177022.456926:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177022.456931:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177022.456931:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177022.456935:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177022.456937:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:1.0:1786177022.456942:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856326912:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1786177022.456942:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.456948:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177022.456952:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177022.456954:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177022.456956:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786177022.456959:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000010:1.0:1786177022.456960:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177022.456963:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1786177022.456964:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177022.456968:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177022.456971:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177022.456972:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177022.456981:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00000010:2.0:1786177022.456981:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000020:00000040:2.0:1786177022.456987:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177022.456989:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177022.456990:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177022.456991:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000001:2.0:1786177022.456992:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177022.456994:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177022.456997:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177022.457000:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856326912, portal 25
02000000:00000010:2.0:1786177022.457002:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:1.0:1786177022.457004:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177022.457007:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454181632 : -114548255369984 : ffff97d1a7edd500)
00000100:00000001:2.0:1786177022.457007:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177022.457015:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.457022:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177022.457025:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177022.457028:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177022.457030:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856326912, offset 0
00000100:00100000:2.0:1786177022.457034:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177022.457037:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177022.457045:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786177022.457048:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177022.457049:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177022.457055:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786177022.457056:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177022.457064:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786177022.457067:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177022.457071:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:2.0:1786177022.457071:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786177022.457076:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1786177022.457076:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177022.457078:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1786177022.457081:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177022.457084:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254725:254725:256:4294967295] 192.168.202.110@tcp LPNI seq info [254714:254714:8:4294967295]
00000100:00080000:2.0:1786177022.457086:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786177022.457091:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177022.457092:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177022.457096:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7245
00000400:00000200:2.0:1786177022.457115:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177022.457116:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1786177022.457122:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177022.457126:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:1.0:1786177022.457127:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177022.457132:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.457134:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:1.0:1786177022.457136:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000100:00000040:2.0:1786177022.457136:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.457142:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177022.457144:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177022.457145:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000800:00000200:1.0:1786177022.457148:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177022.457152:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786177022.457156:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786177022.457159:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:1.1:1786177022.457160:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000040:2.0:1786177022.457162:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000800:00000100:1.0:1786177022.457167:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:2.0:1786177022.457167:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000010:1.0:1786177022.457172:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000020:00000001:2.0:1786177022.457172:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786177022.457177:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.457180:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177022.457181:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177022.457183:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1786177022.457183:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177022.457187:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:1.0:1786177022.457191:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177022.457196:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177022.457198:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177022.457201:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177022.457205:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786177022.457210:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177022.457214:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786177022.457214:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177022.457218:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1786177022.457221:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177022.457223:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177022.457227:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177022.457231:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177022.457234:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1786177022.457236:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1786177022.457237:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.457239:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177022.457240:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177022.457244:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000100:00000001:1.0:1786177022.457245:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786177022.457247:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:1.0:1786177022.457252:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177022.457252:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177022.457255:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.457257:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301921997
00000100:00000010:1.0:1786177022.457262:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b11e0.
00000100:00000001:1.0:1786177022.457268:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177022.457271:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177022.457274:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786177022.457276:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301921997
02000000:00000001:2.0:1786177022.457276:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177022.457281:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000010:1.0:1786177022.457284:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1f00.
00000100:00000001:2.0:1786177022.457285:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.457290:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00100000:2.0:1786177022.457292:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177022.457297:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177022.457299:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177022.457303:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.457304:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177022.457307:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.457309:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1786177022.457316:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177022.457319:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177022/real 1786177022]  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786177022.457319:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:1.0:1786177022.457330:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177022.457333:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177022.457336:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000100:00080000:2.0:1786177022.457337:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:1.0:1786177022.457341:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000100:00000001:2.0:1786177022.457341:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177022.457342:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1786177022.457346:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:2.0:1786177022.457347:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786177022.457352:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00080000:2.0:1786177022.457353:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7245
00000100:00000200:1.0:1786177022.457360:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786177022.457361:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177022.457370:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177022.457376:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:1.0:1786177022.457379:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177022.457382:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.457388:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177022.457393:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177022.457395:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177022.457397:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177022.457399:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1786177022.457402:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786177022.457404:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:2.0:1786177022.457406:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:1.0:1786177022.457409:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1786177022.457410:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177022.457415:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00100000:1.0:1786177022.457419:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177022.457420:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177022.457427:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786177022.457436:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177022.457437:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177022.457442:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177022.457446:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177022.457449:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.457468:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177022.457473:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:1.0:1786177022.457479:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177022.457481:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177022.457483:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.457487:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177022.457488:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.457489:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.457495:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177022.457496:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177022.457498:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177022.457502:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177022.457503:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177022.457506:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177022.457530:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856326912:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177022.457534:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.457536:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177022.457541:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd500 x1872934856326912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177022.457541:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177022.457546:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177022.457551:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177022.457555:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177022.457560:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000008:00100000:2.0:1786177022.457560:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177022.457565:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177022.457566:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:2.0:1786177022.457567:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177022.457572:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
00000100:00000010:2.0:1786177022.457574:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
02000000:00000001:1.0:1786177022.457579:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000040:2.0:1786177022.457579:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177022.457584:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177022.457587:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786177022.457590:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177022.457592:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0800.
02000000:00000001:1.0:1786177022.457597:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177022.457599:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177022.457601:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd500.
00000100:00000001:1.0:1786177022.457604:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177022.457607:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786177022.457607:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177022.457611:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.457613:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177022.457616:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177022.457620:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786177022.457622:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:2.0:1786177022.457622:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.457626:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.457630:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000010:2.0:1786177022.457631:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:1.0:1786177022.457633:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177022.457636:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786177022.457638:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177022.457643:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177022.457649:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177022.457662:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177022.457665:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177022.457668:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1786177022.457669:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1786177022.457672:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177022.457673:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177022.457677:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177022.457678:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177022.457683:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177022.457693:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177022.457695:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177022.457696:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177022.457699:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177022.457702:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177022.457707:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177022.457715:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177022.457717:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786177022.457719:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1786177022.457722:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00080000:2.0:1786177022.457723:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177022.457726:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177022.457729:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1786177022.457730:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000001:2.0:1786177022.457731:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177022.457736:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7245
00000100:00000200:1.0:1786177022.457737:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856327040, portal 10
00000400:00000200:2.0:1786177022.457743:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177022.457744:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000400:00000200:2.0:1786177022.457748:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786177022.457749:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178048 : -114548255373568 : ffff97d1a7edc700)
00000100:00000040:1.0:1786177022.457757:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.457762:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177022.457766:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786177022.457767:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:1.0:1786177022.457768:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177022.457775:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856327040, offset 0
00000100:00000040:2.0:1786177022.457777:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177022.457783:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786177022.457784:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177022.457788:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177022.457793:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177022.457794:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786177022.457799:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177022.457802:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177022.457804:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177022.457806:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177022.457808:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177022.457810:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254726:254726:256:4294967295] 192.168.202.110@tcp LPNI seq info [254715:254715:8:4294967295]
00000400:00000200:1.0:1786177022.457813:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177022.457817:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000001:2.0:1786177022.457820:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000200:1.0:1786177022.457822:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:2.0:1786177022.457823:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000800:00000010:1.0:1786177022.457826:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000020:00000001:2.0:1786177022.457827:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177022.457831:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000800:00000200:1.0:1786177022.457835:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786177022.457836:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000800:00000200:1.0:1786177022.457840:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:2.0:1786177022.457841:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177022.457851:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:1.1:1786177022.457852:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177022.457853:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177022.457858:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:1.1:1786177022.457858:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:2.0:1786177022.457864:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177022.457869:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000800:00000100:1.0:1786177022.457870:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786177022.457874:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000010:1.0:1786177022.457875:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000100:00000001:2.0:1786177022.457876:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177022.457880:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786177022.457882:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1786177022.457883:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177022.457888:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177022.457892:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1786177022.457896:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:1.0:1786177022.457897:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000008:00100000:2.0:1786177022.457899:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000010:1.0:1786177022.457902:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000008:00000001:2.0:1786177022.457903:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177022.457907:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.457908:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000010:2.0:1786177022.457910:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000100:00000200:1.0:1786177022.457915:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.457923:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177022.457927:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177022.457927:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177022.457933:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177022.457936:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177022.457940:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786177022.457942:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177022.457943:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177022.457946:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.457947:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:2.0:1786177022.457951:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:1.0:1786177022.457954:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177022.457955:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177022.457961:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000100:00100000:2.0:1786177022.457962:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177022.457970:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177022.457975:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177022.457976:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301918218
00000100:00000001:2.0:1786177022.457978:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.457983:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301921997
00000100:00000040:2.0:1786177022.457984:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.457989:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301918218
00000100:00000001:2.0:1786177022.457993:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.457995:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301921997
00000100:00000040:2.0:1786177022.457998:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7255)
00000100:00000001:1.0:1786177022.458001:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177022.458005:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.458007:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177022.458015:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177022/real 1786177022]  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177022.458025:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177022.458026:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177022.458028:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177022.458031:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177022.458033:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177022.458035:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177022.458038:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177022.458044:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458049:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177022.458050:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458051:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177022.458052:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177022.458055:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458060:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177022.458063:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458068:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177022.458070:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458071:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177022.458072:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458074:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177022.458076:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177022.458078:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177022.458080:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177022.458082:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177022.458086:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177022.458088:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00100000:1.0:1786177022.458095:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177022.458098:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177022.458100:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177022.458100:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177022.458104:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177022.458109:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177022.458113:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177022.458116:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458121:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177022.458123:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177022.458125:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177022.458126:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177022.458127:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177022.458129:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177022.458131:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1786177022.458139:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177022.458141:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177022.458144:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177022.458145:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177022.458147:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177022.458149:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177022.458152:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177022.458154:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177022.458157:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177022.458159:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177022.458161:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177022.458164:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856327168, portal 10
00000100:00000001:1.0:1786177022.458165:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177022.458166:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454180736 : -114548255370880 : ffff97d1a7edd180)
00000100:00000040:1.0:1786177022.458170:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458175:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177022.458176:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177022.458177:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177022.458179:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856327168, offset 0
00000400:00000200:1.0:1786177022.458182:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177022.458184:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177022.458187:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177022.458189:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177022.458191:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177022.458193:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177022.458194:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177022.458197:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254727:254727:256:4294967295] 192.168.202.110@tcp LPNI seq info [254716:254716:8:4294967295]
00000400:00000200:1.0:1786177022.458200:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177022.458204:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177022.458207:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177022.458209:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177022.458212:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177022.458214:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786177022.458217:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177022.458219:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1786177022.458221:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177022.458223:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177022.458225:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177022.458228:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177022.458230:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177022.458231:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177022.458237:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177022.458240:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458249:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177022.458252:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458259:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.458261:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177022.458264:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177022.458267:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177022.458270:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177022.458272:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301918343
00000100:00000001:1.0:1786177022.458275:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301921997
00000100:00000001:1.0:1786177022.458277:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301918343
00000100:00000001:1.0:1786177022.458280:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301921997
00000100:00000001:1.0:1786177022.458282:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177022.458285:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.458287:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177022.458291:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177022/real 1786177022]  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177022.458300:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177022.458301:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177022.458303:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177022.458306:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177022.458308:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177022.458310:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177022.458313:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177022.458319:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458324:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177022.458325:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458326:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177022.458327:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177022.458329:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458334:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177022.458337:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458341:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177022.458343:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458344:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177022.458344:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458345:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177022.458347:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177022.458348:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177022.458351:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177022.458352:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177022.458355:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177022.458360:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177022.458362:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458363:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177022.458365:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc700 x1872934856327040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458370:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177022.458372:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177022.458374:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177022.458377:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177022.458378:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177022.458379:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177022.458381:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177022.458383:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:1.0:1786177022.458386:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177022.458387:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177022.458388:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc700.
00000100:00000001:1.0:1786177022.458391:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177022.458393:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458394:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177022.458399:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd180 x1872934856327168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458406:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177022.458408:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177022.458409:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177022.458412:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177022.458414:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177022.458415:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177022.458417:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177022.458419:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:1.0:1786177022.458421:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177022.458422:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177022.458423:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd180.
00000100:00000001:1.0:1786177022.458424:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177022.458425:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458427:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458430:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177022.458431:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.458432:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177022.458433:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177022.458434:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177022.458437:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458442:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177022.458443:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177022.458444:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177022.458446:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177022.458447:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177022.458465:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177022.458468:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177022.458470:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177022.458471:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177022.458472:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177022.458475:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177022.458476:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177022.458479:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177022.458481:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177022.458484:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177022.458486:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856327296, portal 4
00000100:00000001:1.0:1786177022.458487:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177022.458488:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187904 : -114548255363712 : ffff97d1a7eded80)
00000100:00000040:1.0:1786177022.458493:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458501:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177022.458503:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177022.458504:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177022.458506:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856327296, offset 0
00000400:00000200:1.0:1786177022.458524:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177022.458527:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177022.458530:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177022.458532:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177022.458534:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177022.458536:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177022.458538:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177022.458542:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254728:254728:256:4294967295] 192.168.202.110@tcp LPNI seq info [254717:254717:8:4294967295]
00000400:00000200:1.0:1786177022.458546:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177022.458552:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177022.458557:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177022.458561:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177022.458565:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177022.458568:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786177022.458573:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177022.458576:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1786177022.458580:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177022.458582:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177022.458586:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177022.458589:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177022.458593:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177022.458595:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177022.458598:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177022.458602:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458610:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177022.458614:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458628:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.458630:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177022.458634:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177022.458637:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177022.458641:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177022.458643:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301918219
00000100:00000001:1.0:1786177022.458645:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301921997
00000100:00000001:1.0:1786177022.458647:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301918219
00000100:00000001:1.0:1786177022.458648:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301921997
00000100:00000001:1.0:1786177022.458650:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177022.458651:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.458652:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177022.458655:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177022/real 1786177022]  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177022.458660:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177022.458662:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177022.458663:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177022.458664:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177022.458666:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177022.458667:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177022.458669:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177022.458674:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458679:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177022.458680:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458681:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177022.458682:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177022.458684:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458688:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177022.458692:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458698:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177022.458699:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458701:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177022.458701:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458702:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177022.458703:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177022.458705:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177022.458707:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177022.458709:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177022.458711:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177022.458716:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177022.458719:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177022.458721:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458725:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177022.458726:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177022.458727:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177022.458729:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177022.458731:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177022.458734:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177022.458737:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177022.458739:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177022.458740:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177022.458742:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177022.458745:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177022.458747:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177022.458751:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177022.458754:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177022.458757:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177022.458760:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856327424, portal 4
00000100:00000001:1.0:1786177022.458762:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177022.458764:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:1.0:1786177022.458770:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458777:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177022.458780:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177022.458781:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177022.458783:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856327424, offset 0
00000400:00000200:1.0:1786177022.458786:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177022.458789:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177022.458792:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177022.458794:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177022.458796:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177022.458797:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177022.458799:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177022.458801:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254729:254729:256:4294967295] 192.168.202.110@tcp LPNI seq info [254718:254718:8:4294967295]
00000400:00000200:1.0:1786177022.458804:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177022.458808:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177022.458811:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177022.458813:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177022.458816:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177022.458818:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1786177022.458821:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177022.458823:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1786177022.458825:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177022.458827:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177022.458829:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177022.458832:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177022.458834:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177022.458835:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177022.458836:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177022.458839:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458844:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177022.458846:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458850:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.458851:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177022.458853:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177022.458856:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177022.458858:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177022.458859:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301918227
00000100:00000001:1.0:1786177022.458861:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301921997
00000100:00000001:1.0:1786177022.458862:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301918227
00000100:00000001:1.0:1786177022.458864:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301921997
00000100:00000001:1.0:1786177022.458865:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177022.458867:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.458869:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177022.458872:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177022/real 1786177022]  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177022.458880:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177022.458882:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177022.458883:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177022.458886:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177022.458888:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177022.458891:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177022.458894:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177022.458904:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458911:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177022.458913:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458915:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177022.458916:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177022.458920:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458927:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177022.458931:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458938:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177022.458940:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458941:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177022.458942:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458943:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177022.458944:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177022.458946:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177022.458948:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177022.458950:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177022.458952:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177022.458957:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177022.458960:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458961:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177022.458963:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eded80 x1872934856327296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458968:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177022.458969:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177022.458970:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177022.458972:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177022.458973:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177022.458974:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177022.458975:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177022.458977:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:1.0:1786177022.458979:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177022.458980:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177022.458982:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eded80.
00000100:00000001:1.0:1786177022.458985:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177022.458986:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.458988:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177022.458991:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856327424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177022.458998:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177022.458999:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177022.459000:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177022.459003:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177022.459004:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177022.459005:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177022.459007:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177022.459009:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:1.0:1786177022.459011:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177022.459013:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177022.459014:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:1.0:1786177022.459017:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177022.459018:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.459020:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177022.459024:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177022.459026:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177022.459027:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177022.459029:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177023.032621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177023.032627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177023.032631:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177023.032632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177023.032635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177023.032637:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177023.032639:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177023.032642:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177023.032644:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177023.032646:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177023.032647:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177023.032649:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177023.032698:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177023.032701:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177023.032704:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177023.480584:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177023.480590:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177023.480592:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177023.480596:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177023.480599:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177023.480602:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177023.480604:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177023.480606:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177023.480608:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177024.056608:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177024.056614:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177024.056619:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177024.056620:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177024.056623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177024.056624:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177024.056689:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177024.056694:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177024.056698:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177024.504791:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177024.504816:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177024.504819:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177024.504833:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177024.504838:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177024.504842:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177024.504845:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177024.504848:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177024.504852:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177025.080581:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177025.080589:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177025.080595:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177025.080596:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177025.080600:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177025.080601:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177025.080647:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177025.080651:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177025.080653:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177025.528626:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177025.528633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177025.528636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177025.528642:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177025.528646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177025.528650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177025.528653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177025.528657:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177025.528659:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177026.104634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177026.104642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177026.104648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177026.104650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177026.104654:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177026.104655:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177026.104712:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177026.104715:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177026.104718:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177026.296614:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301921997, at 4301922048
00000100:00000040:1.0:1786177026.296614:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301921997, at 4301922048
00000100:00000010:0.0:1786177026.296628:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b11e0.
00000100:00000010:1.0:1786177026.296630:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1f00.
00000100:00000001:3.0:1786177026.552620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177026.552629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177026.552631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177026.552637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177026.552642:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177026.552645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177026.552648:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177026.552651:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177026.552654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177027.128600:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177027.128606:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177027.128611:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177027.128612:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177027.128615:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177027.128616:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177027.128659:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177027.128662:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177027.128665:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177027.576563:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177027.576571:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177027.576574:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177027.576580:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177027.576584:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177027.576587:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177027.576588:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177027.576590:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177027.576593:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177027.576595:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177027.576597:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.576598:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177027.576603:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177027.576608:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177027.576616:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7250
00000400:00000200:0.0:1786177027.576626:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177027.576633:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177027.576636:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177027.576640:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.576644:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177027.576647:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.576650:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177027.576654:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177027.576656:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177027.576658:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177027.576661:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177027.576662:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177027.576664:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177027.576666:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.576667:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177027.576669:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.576670:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177027.576672:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177027.576674:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177027.576676:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177027.576678:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177027.576680:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177027.576682:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.576684:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177027.576685:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177027.576691:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41ad80.
00000020:00000040:0.0:1786177027.576694:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177027.576697:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177027.576699:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.576701:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177027.576704:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177027.576707:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177027.576713:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000100:00000001:0.0:1786177027.576719:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177027.576726:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177027.576739:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177027.576750:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177027.576757:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034e2c676] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177027.576772:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177027.576775:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177027.576777:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177027.576778:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177027.576780:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177027.576782:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177027.576783:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7250
00000400:00000200:0.0:1786177027.576786:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177027.576788:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177027.576790:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177027.576792:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.576793:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177027.576795:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.576796:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177027.576798:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177027.576799:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177027.576800:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177027.576801:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177027.576802:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177027.576804:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177027.576805:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.576807:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177027.576808:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.576809:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177027.576812:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177027.576814:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177027.576817:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177027.576818:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177027.576823:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177027.576826:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:0.0:1786177027.576828:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.576831:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177027.576832:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000008:00100000:0.0:1786177027.576835:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1786177027.576836:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000008:00000001:0.0:1786177027.576840:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.576843:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177027.576847:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41aa00.
00000100:00000040:1.0:1786177027.576849:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177027.576850:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177027.576854:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177027.576856:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.576859:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177027.576860:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177027.576861:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177027.576864:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7800.
00000100:00000001:0.0:1786177027.576866:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177027.576874:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177027.576880:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177027.576883:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.576885:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177027.576888:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177027.576888:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1786177027.576891:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da0255a4] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786177027.576896:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177027.576901:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.576905:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177027.576908:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856327552:192.168.202.110@tcp:250:kworker.0
00000100:00080000:0.0:1786177027.576909:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177027.576918:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1786177027.576918:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1786177027.576922:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177027.576924:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:1.0:1786177027.576924:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177027.576928:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.576929:0:62275:0:(import.c:485:import_select_connection()) Process entered
02000000:00000001:1.0:1786177027.576932:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:0.0:1786177027.576935:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7250
00000400:00000200:0.0:1786177027.576938:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:1.0:1786177027.576939:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
00000400:00000200:0.0:1786177027.576942:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:1.0:1786177027.576943:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177027.576945:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177027.576949:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177027.576952:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177027.576953:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177027.576958:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:1.0:1786177027.576959:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:0.0:1786177027.576960:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177027.576964:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177027.576966:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177027.576967:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177027.576968:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000010:1.0:1786177027.576969:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000020:00000001:0.0:1786177027.576970:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177027.576975:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177027.576977:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.576978:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177027.576980:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:1.0:1786177027.576980:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856327552, portal 25
00000100:00000001:0.0:1786177027.576984:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177027.576987:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000001:1.0:1786177027.576987:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1786177027.576992:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177027.576992:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761666432 : -114547947885184 : ffff97d1ba41ad80)
00000100:00000001:0.0:1786177027.577002:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177027.577004:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177027.577005:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177027.577008:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177027.577012:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.577016:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177027.577019:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1786177027.577022:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000008:00000001:0.0:1786177027.577023:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.577026:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177027.577027:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000010:0.0:1786177027.577030:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419500.
00000400:00000010:1.0:1786177027.577032:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000020:00000040:0.0:1786177027.577034:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000200:1.0:1786177027.577036:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856327552, offset 0
02000000:00000001:0.0:1786177027.577038:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177027.577041:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.577042:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177027.577045:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1786177027.577046:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786177027.577048:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177027.577053:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000400:00000200:1.0:1786177027.577057:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177027.577058:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177027.577065:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:0.0:1786177027.577066:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177027.577073:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1786177027.577076:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177027.577079:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177027.577080:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:0.0:1786177027.577084:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b6718a9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786177027.577086:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177027.577092:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177027.577099:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254730:254730:256:4294967295] 192.168.202.110@tcp LPNI seq info [254719:254719:8:4294967295]
00000100:00000001:0.0:1786177027.577101:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177027.577105:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786177027.577106:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177027.577110:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177027.577113:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177027.577114:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177027.577120:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1786177027.577122:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1786177027.577125:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7250
00000400:00000200:0.0:1786177027.577134:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:1.0:1786177027.577134:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177027.577141:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000010:1.0:1786177027.577142:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000100:00080000:0.0:1786177027.577143:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177027.577147:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.577148:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1786177027.577152:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786177027.577153:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.577157:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:1.0:1786177027.577158:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:0.0:1786177027.577160:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177027.577162:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177027.577165:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177027.577166:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000800:00000200:1.1:1786177027.577167:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:0.0:1786177027.577169:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000200:1.1:1786177027.577172:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:0.0:1786177027.577173:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177027.577179:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000100:1.0:1786177027.577181:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177027.577182:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:1.0:1786177027.577184:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000100:00000040:0.0:1786177027.577186:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.577189:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177027.577191:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000400:00000200:1.0:1786177027.577191:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:0.0:1786177027.577196:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177027.577198:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177027.577201:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786177027.577202:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1786177027.577204:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177027.577208:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.577209:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177027.577211:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:1.0:1786177027.577211:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000008:00000001:0.0:1786177027.577216:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.577217:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177027.577217:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000010:0.0:1786177027.577222:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a680.
00000020:00000040:0.0:1786177027.577225:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:1.0:1786177027.577225:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:0.0:1786177027.577230:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177027.577231:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.577233:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177027.577234:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:1.0:1786177027.577235:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177027.577236:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177027.577241:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5800.
00000100:00000001:0.0:1786177027.577245:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177027.577253:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177027.577255:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177027.577262:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1786177027.577263:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177027.577267:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177027.577275:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000911ee1e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.577277:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.577281:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177027.577287:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.577289:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177027.577294:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177027.577296:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786177027.577297:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301927117
00000100:00080000:0.0:1786177027.577304:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000010:1.0:1786177027.577304:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1f00.
00000100:00000001:0.0:1786177027.577309:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177027.577311:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177027.577311:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1786177027.577318:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301927117
00000100:00000010:1.0:1786177027.577320:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1120.
00000100:00000001:1.0:1786177027.577322:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786177027.577327:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786177027.577327:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177027.577332:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.577335:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00080000:0.0:1786177027.577336:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7250
00000100:00100000:1.0:1786177027.577341:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177027/real 1786177027]  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177027.577346:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177027.577353:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177027.577356:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000001:1.0:1786177027.577356:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:0.0:1786177027.577360:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000001:1.0:1786177027.577360:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177027.577362:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:0.0:1786177027.577364:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786177027.577366:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000100:00000040:0.0:1786177027.577368:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:1.0:1786177027.577370:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:0.0:1786177027.577371:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177027.577375:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:0.0:1786177027.577376:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177027.577379:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177027.577381:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000200:1.0:1786177027.577382:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177027.577383:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177027.577388:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:0.0:1786177027.577392:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786177027.577399:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.577414:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1786177027.577417:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177027.577419:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177027.577424:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177027.577428:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177027.577435:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177027.577435:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177027.577438:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177027.577441:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.577441:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177027.577444:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177027.577449:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177027.577452:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786177027.577457:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177027.577458:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1786177027.577462:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177027.577465:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.577470:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177027.577472:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177027.577476:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177027.577476:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177027.577478:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:0.0:1786177027.577482:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786177027.577482:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177027.577485:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000008:00000001:0.0:1786177027.577490:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177027.577490:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:0.0:1786177027.577495:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177027.577497:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000010:0.0:1786177027.577500:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a300.
00000100:00000001:1.0:1786177027.577501:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177027.577507:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177027.577540:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856327552:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177027.577545:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.577546:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177027.577549:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41ad80 x1872934856327552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.577554:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177027.577557:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177027.577559:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177027.577563:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177027.577564:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177027.577566:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786177027.577567:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177027.577569:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:1.0:1786177027.577572:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000020:00000040:0.0:1786177027.577573:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:1.0:1786177027.577574:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177027.577577:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41ad80.
02000000:00000001:0.0:1786177027.577580:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786177027.577581:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
02000000:00000001:0.0:1786177027.577583:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.577584:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177027.577587:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786177027.577588:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177027.577591:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177027.577594:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.577597:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000010:0.0:1786177027.577598:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4c00.
00000100:00000001:1.0:1786177027.577599:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.577604:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177027.577606:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.577606:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177027.577609:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177027.577615:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177027.577619:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177027.577626:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177027.577629:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177027.577633:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177027.577636:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177027.577640:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1786177027.577640:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.577646:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177027.577647:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327680:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177027.577653:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177027.577655:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:0.0:1786177027.577658:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b0d3569] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786177027.577658:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177027.577660:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177027.577664:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177027.577667:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177027.577673:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177027.577674:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177027.577678:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00000040:0.0:1786177027.577680:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7261)
00000400:00000010:1.0:1786177027.577682:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177027.577686:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856327680, portal 10
00000100:00000001:1.0:1786177027.577688:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177027.577689:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761665536 : -114547947886080 : ffff97d1ba41aa00)
00000100:00000040:1.0:1786177027.577695:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.577700:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177027.577703:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177027.577704:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177027.577706:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856327680, offset 0
00000400:00000200:1.0:1786177027.577709:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177027.577713:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177027.577716:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177027.577719:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177027.577722:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177027.577724:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177027.577727:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177027.577731:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254731:254731:256:4294967295] 192.168.202.110@tcp LPNI seq info [254720:254720:8:4294967295]
00000400:00000200:1.0:1786177027.577735:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177027.577740:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177027.577745:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177027.577747:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177027.577751:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177027.577753:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786177027.577757:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177027.577759:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1786177027.577764:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177027.577766:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177027.577768:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177027.577770:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177027.577772:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177027.577774:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177027.577775:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177027.577777:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.577783:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177027.577784:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.577789:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.577790:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177027.577792:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177027.577795:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786177027.577797:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177027.577798:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301923338
00000100:00000001:1.0:1786177027.577800:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301927117
00000100:00000001:1.0:1786177027.577802:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301923338
00000100:00000001:1.0:1786177027.577803:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301927117
00000100:00000001:1.0:1786177027.577805:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177027.577807:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.577808:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177027.577810:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177027/real 1786177027]  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177027.577816:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177027.577817:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177027.577818:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177027.577819:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177027.577820:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177027.577822:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177027.577824:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177027.577832:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.577839:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177027.577840:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.577842:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177027.577843:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177027.577847:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.577854:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177027.577858:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.577866:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177027.577869:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.577871:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177027.577872:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.577874:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177027.577876:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177027.577880:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177027.577882:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177027.577884:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177027.577887:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177027.577892:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327680:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177027.577895:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177027.577898:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.577902:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177027.577903:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177027.577905:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177027.577906:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177027.577907:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177027.577909:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177027.577911:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177027.577913:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177027.577914:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177027.577915:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177027.577918:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177027.577919:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177027.577922:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177027.577923:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177027.577926:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177027.577928:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856327808, portal 10
00000100:00000001:1.0:1786177027.577929:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177027.577930:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761660160 : -114547947891456 : ffff97d1ba419500)
00000100:00000040:1.0:1786177027.577934:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.577938:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177027.577940:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177027.577941:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177027.577943:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856327808, offset 0
00000400:00000200:1.0:1786177027.577945:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177027.577948:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177027.577951:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177027.577953:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177027.577955:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177027.577956:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177027.577958:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177027.577961:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254732:254732:256:4294967295] 192.168.202.110@tcp LPNI seq info [254721:254721:8:4294967295]
00000400:00000200:1.0:1786177027.577966:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177027.577972:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177027.577977:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177027.577980:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177027.577984:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177027.577987:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786177027.577992:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177027.577995:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1786177027.577999:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177027.578001:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177027.578004:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177027.578008:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177027.578011:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177027.578013:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177027.578015:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177027.578018:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578023:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177027.578025:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578030:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.578031:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177027.578033:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177027.578036:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177027.578038:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177027.578039:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301923463
00000100:00000001:1.0:1786177027.578041:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301927117
00000100:00000001:1.0:1786177027.578043:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301923463
00000100:00000001:1.0:1786177027.578044:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301927117
00000100:00000001:1.0:1786177027.578046:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177027.578047:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.578048:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177027.578050:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177027/real 1786177027]  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177027.578055:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177027.578056:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177027.578057:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177027.578059:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177027.578060:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177027.578062:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177027.578064:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177027.578069:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578073:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177027.578073:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578074:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177027.578075:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177027.578077:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578081:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177027.578084:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578089:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177027.578090:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578092:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177027.578092:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578093:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177027.578095:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177027.578096:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177027.578098:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177027.578099:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177027.578102:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177027.578107:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177027.578110:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177027.578112:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578115:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177027.578116:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177027.578118:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177027.578119:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177027.578120:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177027.578122:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177027.578124:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177027.578125:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177027.578126:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177027.578127:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177027.578129:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177027.578130:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177027.578133:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177027.578135:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177027.578137:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177027.578139:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856327936, portal 4
00000100:00000001:1.0:1786177027.578140:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177027.578141:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761664640 : -114547947886976 : ffff97d1ba41a680)
00000100:00000040:1.0:1786177027.578145:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578149:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177027.578150:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177027.578152:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177027.578153:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856327936, offset 0
00000400:00000200:1.0:1786177027.578156:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177027.578158:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177027.578160:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177027.578162:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177027.578164:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177027.578165:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177027.578167:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177027.578169:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254733:254733:256:4294967295] 192.168.202.110@tcp LPNI seq info [254722:254722:8:4294967295]
00000400:00000200:1.0:1786177027.578172:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177027.578176:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177027.578179:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177027.578181:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177027.578183:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177027.578185:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786177027.578188:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177027.578190:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1786177027.578192:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177027.578194:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177027.578196:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177027.578199:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177027.578202:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177027.578204:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177027.578207:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177027.578210:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578218:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177027.578221:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578228:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.578230:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177027.578233:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177027.578237:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177027.578240:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177027.578242:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301923338
00000100:00000001:1.0:1786177027.578245:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301927117
00000100:00000001:1.0:1786177027.578247:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301923338
00000100:00000001:1.0:1786177027.578250:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301927117
00000100:00000001:1.0:1786177027.578251:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177027.578253:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.578254:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177027.578257:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177027/real 1786177027]  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177027.578263:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177027.578264:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177027.578265:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177027.578267:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177027.578268:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177027.578270:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177027.578272:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177027.578278:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578282:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177027.578284:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578285:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177027.578286:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177027.578288:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578292:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177027.578294:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578298:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177027.578300:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578301:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177027.578302:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578303:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177027.578304:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177027.578306:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177027.578308:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177027.578309:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177027.578312:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177027.578317:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856327936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177027.578320:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578321:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177027.578323:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41aa00 x1872934856327680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578327:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177027.578328:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177027.578330:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177027.578332:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177027.578333:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177027.578334:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177027.578336:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177027.578337:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7800.
02000000:00000001:1.0:1786177027.578339:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177027.578339:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177027.578341:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41aa00.
00000100:00000001:1.0:1786177027.578342:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177027.578343:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578344:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177027.578346:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419500 x1872934856327808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578351:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177027.578351:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177027.578352:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177027.578354:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177027.578355:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177027.578355:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177027.578356:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177027.578357:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:1.0:1786177027.578358:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177027.578359:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177027.578360:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419500.
00000100:00000001:1.0:1786177027.578361:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177027.578362:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578363:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177027.578364:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a680 x1872934856327936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578368:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177027.578369:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177027.578370:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177027.578371:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177027.578372:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177027.578372:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177027.578374:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177027.578375:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5800.
02000000:00000001:1.0:1786177027.578376:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177027.578377:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177027.578378:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a680.
00000100:00000001:1.0:1786177027.578379:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177027.578380:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578381:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578383:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177027.578384:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.578385:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177027.578386:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177027.578387:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177027.578389:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578394:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177027.578395:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177027.578396:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177027.578397:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177027.578399:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177027.578418:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856328064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177027.578420:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177027.578422:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177027.578423:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177027.578424:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177027.578426:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177027.578428:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177027.578430:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177027.578432:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177027.578434:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177027.578436:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856328064, portal 4
00000100:00000001:1.0:1786177027.578437:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177027.578438:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761663744 : -114547947887872 : ffff97d1ba41a300)
00000100:00000040:1.0:1786177027.578443:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578452:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177027.578453:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177027.578455:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177027.578456:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856328064, offset 0
00000400:00000200:1.0:1786177027.578459:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177027.578461:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177027.578464:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177027.578466:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177027.578467:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177027.578469:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177027.578470:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177027.578473:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254734:254734:256:4294967295] 192.168.202.110@tcp LPNI seq info [254723:254723:8:4294967295]
00000400:00000200:1.0:1786177027.578476:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177027.578480:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177027.578482:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177027.578484:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177027.578487:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177027.578489:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1786177027.578491:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177027.578493:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1786177027.578496:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177027.578497:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177027.578500:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177027.578502:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177027.578504:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177027.578505:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177027.578507:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177027.578530:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578536:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177027.578539:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578548:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.578549:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177027.578552:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177027.578554:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177027.578557:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177027.578558:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301923347
00000100:00000001:1.0:1786177027.578560:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301927117
00000100:00000001:1.0:1786177027.578562:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301923347
00000100:00000001:1.0:1786177027.578563:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301927117
00000100:00000001:1.0:1786177027.578564:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177027.578566:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.578567:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177027.578569:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177027/real 1786177027]  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177027.578574:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177027.578575:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177027.578576:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177027.578577:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177027.578579:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177027.578580:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177027.578582:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177027.578587:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578591:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177027.578592:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578593:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177027.578593:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177027.578596:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578600:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177027.578603:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578607:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177027.578609:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578610:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177027.578610:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578612:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177027.578613:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177027.578614:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177027.578616:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177027.578617:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177027.578620:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177027.578624:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856328064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177027.578626:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578627:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177027.578629:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a300 x1872934856328064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177027.578633:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177027.578634:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177027.578636:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177027.578637:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177027.578638:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177027.578639:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177027.578641:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177027.578642:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:1.0:1786177027.578643:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177027.578645:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177027.578646:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a300.
00000100:00000001:1.0:1786177027.578647:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177027.578648:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578649:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177027.578652:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177027.578653:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177027.578654:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177027.578655:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177028.152642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177028.152651:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177028.152658:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177028.152659:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177028.152662:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177028.152663:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177028.152727:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177028.152731:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177028.152733:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177028.600566:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177028.600572:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177028.600574:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177028.600578:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177028.600581:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177028.600584:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177028.600586:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177028.600588:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177028.600590:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177029.176594:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177029.176600:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177029.176603:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177029.176604:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177029.176607:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177029.176608:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177029.176668:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177029.176671:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177029.176674:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.0:1786177029.496603:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786177029.496612:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786177029.496618:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:0.0:1786177029.496795:0:2384:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1786177029.496799:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786177029.496809:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1786177029.496814:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786177029.496816:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177029.624650:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177029.624657:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177029.624659:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177029.624665:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177029.624668:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177029.624672:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177029.624676:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177029.624679:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177029.624683:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177030.200588:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177030.200593:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177030.200597:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177030.200598:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177030.200600:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177030.200601:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177030.200650:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177030.200654:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177030.200656:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177030.648583:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177030.648591:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177030.648593:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177030.648600:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177030.648604:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177030.648607:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177030.648609:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177030.648611:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177030.648613:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177031.224613:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177031.224620:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177031.224622:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177031.224625:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177031.224628:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177031.224630:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177031.224632:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177031.224634:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177031.224636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177031.224638:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177031.224639:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177031.224640:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177031.224641:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177031.224642:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177031.224644:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177031.224645:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177031.224647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177031.224648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177031.224697:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177031.224701:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177031.224711:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177031.416629:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301927117, at 4301927168
00000100:00000040:1.0:1786177031.416629:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301927117, at 4301927168
00000100:00000010:0.0:1786177031.416644:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1f00.
00000100:00000010:1.0:1786177031.416646:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1120.
00000100:00000001:3.0:1786177031.672650:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177031.672660:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177031.672662:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177031.672667:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177031.672670:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177031.672674:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177031.672678:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177031.672682:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177031.672686:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177032.248617:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177032.248625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177032.248630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177032.248631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177032.248633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177032.248634:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177032.248696:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177032.248700:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177032.248703:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177032.696573:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177032.696584:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177032.696587:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177032.696592:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1786177032.696595:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:0.0:1786177032.696601:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177032.696602:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177032.696604:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177032.696608:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177032.696609:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177032.696615:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177032.696615:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177032.696621:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177032.696626:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7256
00000100:00000001:3.0:1786177032.696627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177032.696636:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177032.696642:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177032.696646:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177032.696649:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.696652:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177032.696654:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.696656:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177032.696660:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177032.696661:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177032.696664:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177032.696667:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177032.696669:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177032.696670:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177032.696673:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.696674:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177032.696676:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.696677:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177032.696679:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177032.696682:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177032.696683:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177032.696686:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177032.696689:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177032.696692:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.696693:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177032.696695:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177032.696704:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419c00.
00000020:00000040:0.0:1786177032.696709:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177032.696714:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177032.696716:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.696720:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177032.696723:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177032.696730:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177032.696740:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
00000100:00000001:0.0:1786177032.696749:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177032.696761:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177032.696775:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177032.696781:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177032.696787:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cf890c1c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177032.696812:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177032.696815:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177032.696823:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177032.696824:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177032.696827:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177032.696830:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177032.696833:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7256
00000400:00000200:0.0:1786177032.696837:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177032.696841:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177032.696844:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177032.696848:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.696851:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177032.696854:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.696857:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177032.696860:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177032.696861:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177032.696864:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786177032.696864:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1786177032.696870:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:1.0:1786177032.696870:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177032.696874:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1786177032.696876:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177032.696882:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:1.0:1786177032.696884:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177032.696887:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.696892:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177032.696896:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.696901:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177032.696903:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000040:0.0:1786177032.696904:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786177032.696905:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177032.696909:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:1.0:1786177032.696912:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1786177032.696913:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177032.696916:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177032.696916:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177032.696920:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177032.696922:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:1.0:1786177032.696924:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856328192:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1786177032.696927:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.696930:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177032.696930:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:0.0:1786177032.696937:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786177032.696937:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:0.0:1786177032.696942:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177032.696942:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.696947:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177032.696951:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41bb80.
00000020:00000040:0.0:1786177032.696952:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177032.696954:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177032.696956:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177032.696957:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1786177032.696958:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177032.696960:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177032.696962:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177032.696964:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000010:0.0:1786177032.696967:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4000.
02000000:00000001:1.0:1786177032.696968:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.696975:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177032.696979:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:0.0:1786177032.696983:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1786177032.696986:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:0.0:1786177032.696993:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177032.696996:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177032.696999:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9983a05] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1786177032.697000:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177032.697011:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856328192, portal 25
00000100:00000001:1.0:1786177032.697016:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177032.697017:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.697019:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761661952 : -114547947889664 : ffff97d1ba419c00)
00000100:00080000:0.0:1786177032.697021:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177032.697025:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000040:1.0:1786177032.697027:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177032.697028:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177032.697031:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177032.697034:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177032.697038:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7256
00000100:00000001:1.0:1786177032.697043:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:0.0:1786177032.697048:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1786177032.697049:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177032.697054:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000400:00000200:0.0:1786177032.697057:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000200:1.0:1786177032.697058:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856328192, offset 0
00000100:00080000:0.0:1786177032.697064:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786177032.697070:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:0.0:1786177032.697073:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697077:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177032.697080:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697084:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786177032.697085:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:0.0:1786177032.697089:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177032.697091:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177032.697093:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177032.697095:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177032.697098:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786177032.697099:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:0.0:1786177032.697101:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177032.697107:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697108:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177032.697111:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177032.697111:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1786177032.697116:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177032.697119:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:1.0:1786177032.697122:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:0.0:1786177032.697123:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697126:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177032.697130:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1786177032.697130:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1786177032.697134:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786177032.697138:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:0.0:1786177032.697139:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.697143:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177032.697144:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177032.697147:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.697149:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177032.697152:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418380.
00000020:00000040:0.0:1786177032.697155:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:1.0:1786177032.697157:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254735:254735:256:4294967295] 192.168.202.110@tcp LPNI seq info [254724:254724:8:4294967295]
02000000:00000001:0.0:1786177032.697159:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177032.697162:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.697167:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1786177032.697170:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786177032.697171:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177032.697176:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177032.697181:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000400:00000200:1.0:1786177032.697188:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177032.697190:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786177032.697199:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00100000:0.0:1786177032.697200:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:1.0:1786177032.697206:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000100:00000001:0.0:1786177032.697209:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177032.697212:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177032.697215:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e6b901d7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1786177032.697218:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177032.697225:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:0.0:1786177032.697227:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177032.697231:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.1:1786177032.697235:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1786177032.697237:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177032.697240:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:1.1:1786177032.697241:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00080000:0.0:1786177032.697242:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177032.697247:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000800:00000100:1.0:1786177032.697249:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177032.697252:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7256
00000800:00000010:1.0:1786177032.697254:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:0.0:1786177032.697258:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177032.697262:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1786177032.697263:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:0.0:1786177032.697265:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177032.697269:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697271:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177032.697273:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697275:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786177032.697278:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1786177032.697279:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177032.697281:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177032.697285:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177032.697287:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177032.697288:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:1.0:1786177032.697289:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000020:00000001:0.0:1786177032.697290:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000010:1.0:1786177032.697293:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000040:0.0:1786177032.697296:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697299:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177032.697300:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:0.0:1786177032.697301:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697304:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177032.697307:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000200:1.0:1786177032.697308:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177032.697312:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697316:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177032.697319:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177032.697323:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177032.697327:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.697331:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177032.697334:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177032.697337:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.697338:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177032.697339:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177032.697344:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b800.
00000100:00000040:1.0:1786177032.697344:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177032.697348:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177032.697368:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786177032.697368:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177032.697371:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.697372:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177032.697374:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177032.697376:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177032.697378:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177032.697382:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:1.0:1786177032.697382:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.697388:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177032.697392:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177032.697393:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177032.697397:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301932237
00000100:00000010:1.0:1786177032.697404:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1120.
00000100:00000001:0.0:1786177032.697405:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177032.697408:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177032.697411:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000192ee9a0] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.697414:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786177032.697422:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.697423:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301932237
00000100:00080000:0.0:1786177032.697425:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:1.0:1786177032.697429:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1b40.
00000100:00080000:0.0:1786177032.697430:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177032.697432:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177032.697435:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177032.697437:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786177032.697437:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00080000:0.0:1786177032.697442:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7256
00000100:00000001:1.0:1786177032.697444:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1786177032.697450:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177032.697450:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.697456:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000400:00000200:0.0:1786177032.697457:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177032.697462:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00100000:1.0:1786177032.697467:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177032/real 1786177032]  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177032.697469:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697475:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177032.697480:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697485:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177032.697490:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000001:1.0:1786177032.697491:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:0.0:1786177032.697492:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177032.697496:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000001:1.0:1786177032.697496:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:0.0:1786177032.697498:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177032.697501:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000010:1.0:1786177032.697501:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000020:00000001:0.0:1786177032.697507:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:1.0:1786177032.697508:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000100:00000040:0.0:1786177032.697532:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697535:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177032.697538:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697540:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177032.697544:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177032.697548:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177032.697551:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177032.697553:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
02000000:00000001:0.0:1786177032.697556:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786177032.697558:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:0.0:1786177032.697561:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1786177032.697565:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177032.697566:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.697573:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177032.697578:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000200:1.0:1786177032.697579:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1786177032.697582:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.697587:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177032.697593:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418000.
00000100:00000001:1.0:1786177032.697594:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000040:0.0:1786177032.697596:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:1.0:1786177032.697598:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177032.697600:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786177032.697602:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:0.0:1786177032.697603:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177032.697606:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786177032.697606:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177032.697610:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177032.697614:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177032.697614:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786177032.697621:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000100:00000001:1.0:1786177032.697625:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177032.697626:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177032.697632:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177032.697634:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177032.697654:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177032.697656:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177032.697658:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177032.697659:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.697661:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:0.0:1786177032.697665:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002e10e4f5] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.697670:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.697675:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177032.697677:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177032.697679:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177032.697684:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177032.697685:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1786177032.697687:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177032.697691:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177032.697699:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7266)
00000100:00100000:1.0:1786177032.697700:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856328192:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177032.697708:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.697710:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177032.697713:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419c00 x1872934856328192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.697718:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177032.697721:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177032.697723:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177032.697726:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177032.697728:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177032.697731:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786177032.697732:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177032.697735:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
02000000:00000001:1.0:1786177032.697738:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177032.697740:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177032.697742:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419c00.
00000100:00000001:1.0:1786177032.697744:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177032.697745:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.697747:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.697754:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177032.697755:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.697756:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177032.697758:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177032.697759:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177032.697761:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.697766:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177032.697767:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177032.697769:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177032.697775:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177032.697776:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177032.697778:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856328320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177032.697781:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177032.697782:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177032.697784:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177032.697786:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177032.697788:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177032.697789:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177032.697793:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177032.697795:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177032.697797:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177032.697801:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856328320, portal 10
00000100:00000001:1.0:1786177032.697803:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177032.697805:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761670016 : -114547947881600 : ffff97d1ba41bb80)
00000100:00000040:1.0:1786177032.697811:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.697819:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177032.697821:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177032.697824:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177032.697827:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856328320, offset 0
00000400:00000200:1.0:1786177032.697832:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177032.697836:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177032.697841:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177032.697844:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177032.697847:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177032.697850:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177032.697853:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177032.697856:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254736:254736:256:4294967295] 192.168.202.110@tcp LPNI seq info [254725:254725:8:4294967295]
00000400:00000200:1.0:1786177032.697859:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177032.697865:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177032.697868:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177032.697871:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177032.697874:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177032.697876:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786177032.697880:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177032.697881:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786177032.697884:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177032.697886:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177032.697888:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177032.697891:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177032.697893:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177032.697895:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177032.697896:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177032.697899:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.697905:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177032.697908:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.697917:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.697918:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177032.697920:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177032.697924:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786177032.697926:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177032.697928:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301928458
00000100:00000001:1.0:1786177032.697930:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301932237
00000100:00000001:1.0:1786177032.697932:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301928458
00000100:00000001:1.0:1786177032.697934:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301932237
00000100:00000001:1.0:1786177032.697935:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177032.697936:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.697938:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177032.697940:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177032/real 1786177032]  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177032.697945:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177032.697946:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177032.697948:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177032.697949:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177032.697951:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177032.697952:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177032.697954:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177032.697960:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.697964:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177032.697965:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.697966:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177032.697967:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177032.697969:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.697974:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177032.697977:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.697981:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177032.697983:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.697984:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177032.697985:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.697986:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177032.697987:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177032.697989:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177032.697991:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177032.697992:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177032.697994:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177032.697999:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856328320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177032.698001:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177032.698003:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698008:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177032.698009:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177032.698010:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177032.698012:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177032.698013:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177032.698015:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856328448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177032.698017:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177032.698018:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177032.698020:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177032.698021:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177032.698024:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177032.698025:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177032.698027:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177032.698029:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177032.698031:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177032.698033:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856328448, portal 10
00000100:00000001:1.0:1786177032.698034:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177032.698036:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761655680 : -114547947895936 : ffff97d1ba418380)
00000100:00000040:1.0:1786177032.698040:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698044:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177032.698046:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177032.698047:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177032.698049:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856328448, offset 0
00000400:00000200:1.0:1786177032.698051:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177032.698054:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177032.698056:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177032.698058:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177032.698060:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177032.698061:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177032.698063:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177032.698065:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254737:254737:256:4294967295] 192.168.202.110@tcp LPNI seq info [254726:254726:8:4294967295]
00000400:00000200:1.0:1786177032.698068:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177032.698074:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177032.698079:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177032.698082:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177032.698086:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177032.698089:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786177032.698094:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177032.698097:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786177032.698101:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177032.698104:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177032.698107:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177032.698111:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177032.698114:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177032.698116:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177032.698118:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177032.698122:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698132:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177032.698135:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698142:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.698144:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177032.698147:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177032.698150:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177032.698154:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177032.698156:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301928583
00000100:00000001:1.0:1786177032.698159:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301932237
00000100:00000001:1.0:1786177032.698162:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301928583
00000100:00000001:1.0:1786177032.698164:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301932237
00000100:00000001:1.0:1786177032.698167:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177032.698169:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.698171:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177032.698176:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177032/real 1786177032]  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177032.698184:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177032.698185:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177032.698186:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177032.698188:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177032.698190:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177032.698192:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177032.698194:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177032.698200:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698204:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177032.698205:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698206:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177032.698207:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177032.698209:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698214:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177032.698216:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698221:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177032.698222:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698223:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177032.698224:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698225:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177032.698226:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177032.698228:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177032.698230:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177032.698231:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177032.698234:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177032.698238:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856328448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177032.698240:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177032.698244:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698248:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177032.698250:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177032.698251:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177032.698253:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177032.698254:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177032.698255:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856328576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177032.698258:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177032.698259:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177032.698260:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177032.698262:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177032.698264:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177032.698266:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177032.698268:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177032.698270:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177032.698272:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177032.698275:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856328576, portal 4
00000100:00000001:1.0:1786177032.698276:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177032.698277:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761669120 : -114547947882496 : ffff97d1ba41b800)
00000100:00000040:1.0:1786177032.698282:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698286:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177032.698288:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177032.698289:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177032.698291:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856328576, offset 0
00000400:00000200:1.0:1786177032.698294:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177032.698296:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177032.698299:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177032.698301:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177032.698303:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177032.698305:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177032.698306:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177032.698309:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254738:254738:256:4294967295] 192.168.202.110@tcp LPNI seq info [254727:254727:8:4294967295]
00000400:00000200:1.0:1786177032.698312:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177032.698317:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177032.698320:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177032.698323:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177032.698325:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177032.698327:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786177032.698330:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177032.698332:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786177032.698335:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177032.698336:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177032.698338:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177032.698340:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177032.698342:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177032.698344:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177032.698345:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177032.698348:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698368:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177032.698371:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698375:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.698376:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177032.698379:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177032.698382:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177032.698384:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177032.698386:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301928458
00000100:00000001:1.0:1786177032.698388:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301932237
00000100:00000001:1.0:1786177032.698389:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301928458
00000100:00000001:1.0:1786177032.698391:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301932237
00000100:00000001:1.0:1786177032.698392:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177032.698394:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.698395:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177032.698398:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177032/real 1786177032]  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177032.698403:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177032.698404:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177032.698405:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177032.698406:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177032.698407:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177032.698409:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177032.698411:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177032.698417:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698420:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177032.698421:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698422:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177032.698423:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177032.698425:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698431:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177032.698435:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698441:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177032.698443:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698444:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177032.698446:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698448:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177032.698450:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177032.698453:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177032.698456:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177032.698459:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177032.698464:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177032.698473:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856328576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177032.698478:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177032.698481:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698489:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177032.698490:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177032.698493:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177032.698494:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177032.698496:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177032.698500:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856328704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177032.698503:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177032.698505:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177032.698507:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177032.698529:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177032.698534:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177032.698536:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177032.698539:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177032.698540:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177032.698543:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177032.698545:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856328704, portal 4
00000100:00000001:1.0:1786177032.698547:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177032.698548:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761654784 : -114547947896832 : ffff97d1ba418000)
00000100:00000040:1.0:1786177032.698553:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698558:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177032.698560:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177032.698562:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177032.698563:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856328704, offset 0
00000400:00000200:1.0:1786177032.698566:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177032.698569:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177032.698572:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177032.698574:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177032.698576:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177032.698577:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177032.698579:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177032.698581:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254739:254739:256:4294967295] 192.168.202.110@tcp LPNI seq info [254728:254728:8:4294967295]
00000400:00000200:1.0:1786177032.698584:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177032.698588:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177032.698591:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177032.698593:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177032.698596:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177032.698598:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1786177032.698601:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177032.698603:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1786177032.698606:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177032.698607:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177032.698609:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177032.698612:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177032.698614:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177032.698615:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177032.698617:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177032.698620:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698625:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177032.698627:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698632:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.698633:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177032.698636:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177032.698638:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177032.698640:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177032.698642:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301928467
00000100:00000001:1.0:1786177032.698643:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301932237
00000100:00000001:1.0:1786177032.698645:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301928467
00000100:00000001:1.0:1786177032.698647:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301932237
00000100:00000001:1.0:1786177032.698648:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177032.698649:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.698651:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177032.698653:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177032/real 1786177032]  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177032.698657:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177032.698658:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177032.698659:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177032.698661:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177032.698662:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177032.698663:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177032.698665:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177032.698671:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698674:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177032.698675:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698676:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177032.698677:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177032.698679:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698683:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177032.698686:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698691:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177032.698693:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698694:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177032.698695:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698696:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177032.698698:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177032.698699:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177032.698701:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177032.698702:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177032.698704:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177032.698709:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856328704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177032.698711:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698712:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177032.698714:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41bb80 x1872934856328320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698718:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177032.698719:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177032.698721:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177032.698723:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177032.698724:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177032.698725:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177032.698726:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177032.698727:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4000.
02000000:00000001:1.0:1786177032.698729:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177032.698731:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177032.698732:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41bb80.
00000100:00000001:1.0:1786177032.698734:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177032.698734:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698736:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177032.698738:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418380 x1872934856328448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698743:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177032.698744:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177032.698745:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177032.698747:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177032.698747:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177032.698748:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177032.698749:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177032.698750:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:1.0:1786177032.698751:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177032.698752:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177032.698753:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418380.
00000100:00000001:1.0:1786177032.698755:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177032.698756:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698757:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177032.698759:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b800 x1872934856328576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698763:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177032.698763:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177032.698764:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177032.698765:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177032.698766:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177032.698767:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177032.698768:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177032.698769:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:1.0:1786177032.698770:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177032.698771:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177032.698772:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b800.
00000100:00000001:1.0:1786177032.698773:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177032.698774:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698775:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177032.698777:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418000 x1872934856328704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177032.698785:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177032.698786:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177032.698787:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177032.698790:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177032.698791:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177032.698791:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177032.698792:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177032.698793:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:1.0:1786177032.698794:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177032.698795:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177032.698796:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418000.
00000100:00000001:1.0:1786177032.698797:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177032.698798:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698799:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177032.698803:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177032.698804:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177032.698806:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177032.698807:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177033.272751:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177033.272762:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177033.272771:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177033.272772:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177033.272776:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177033.272781:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177033.272784:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177033.272789:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177033.272794:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177033.272810:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177033.272813:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177033.272816:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177033.273558:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177033.273565:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177033.273571:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177033.720619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177033.720628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177033.720630:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177033.720636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177033.720640:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177033.720645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177033.720648:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177033.720651:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177033.720654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177034.296623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177034.296633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177034.296641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177034.296642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177034.296647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177034.296648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177034.296709:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177034.296713:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177034.296717:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177034.744644:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177034.744652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177034.744654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177034.744659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177034.744663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177034.744666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177034.744669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177034.744672:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177034.744674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177035.320677:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177035.320686:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177035.320697:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177035.320698:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177035.320701:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177035.320703:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177035.320810:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177035.320817:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177035.320822:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177035.768701:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177035.768712:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177035.768719:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177035.768728:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177035.768733:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177035.768739:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177035.768743:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177035.768746:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177035.768750:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177036.344695:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177036.344704:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177036.344711:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177036.344713:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177036.344717:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177036.344718:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177036.344791:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177036.344795:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177036.344799:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177036.536621:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301932237, at 4301932288
00000100:00000040:1.0:1786177036.536631:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301932237, at 4301932288
00000100:00000010:0.0:1786177036.536653:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1120.
00000100:00000010:1.0:1786177036.536656:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1b40.
00000100:00000001:3.0:1786177036.792669:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177036.792681:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177036.792683:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177036.792691:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177036.792695:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177036.792701:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177036.792705:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177036.792708:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177036.792712:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177037.368655:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177037.368663:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177037.368668:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177037.368669:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177037.368672:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177037.368673:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177037.368759:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177037.368764:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177037.368768:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177037.816646:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177037.816655:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:2.0:1786177037.816656:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177037.816658:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177037.816666:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786177037.816668:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177037.816671:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177037.816673:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177037.816676:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177037.816679:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177037.816680:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177037.816685:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177037.816687:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177037.816689:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177037.816697:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7261
00000400:00000200:2.0:1786177037.816707:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177037.816715:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177037.816720:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177037.816724:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.816728:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177037.816731:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.816734:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177037.816739:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177037.816742:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177037.816746:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177037.816750:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177037.816753:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177037.816757:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177037.816760:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.816762:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177037.816764:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.816766:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177037.816770:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177037.816773:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177037.816777:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177037.816781:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177037.816785:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177037.816789:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177037.816792:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177037.816794:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177037.816805:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786177037.816809:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177037.816815:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177037.816817:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177037.816821:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177037.816825:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177037.816831:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177037.816841:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786177037.816852:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177037.816864:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177037.816877:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177037.816881:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177037.816886:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177037.816912:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177037.816915:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177037.816918:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177037.816919:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177037.816920:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177037.816922:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177037.816924:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7261
00000400:00000200:2.0:1786177037.816927:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177037.816930:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177037.816932:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177037.816934:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.816935:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177037.816937:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.816939:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177037.816941:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177037.816942:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177037.816943:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177037.816945:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177037.816947:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177037.816949:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177037.816951:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.816953:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177037.816955:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177037.816957:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177037.816957:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177037.816961:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786177037.816964:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1786177037.816965:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177037.816969:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.816970:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:2.0:1786177037.816972:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177037.816979:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177037.816982:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177037.816983:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177037.816984:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177037.816990:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177037.816996:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177037.817001:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.817005:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000010:2.0:1786177037.817007:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000100:00000001:1.0:1786177037.817009:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786177037.817011:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177037.817016:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177037.817017:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177037.817018:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1786177037.817019:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177037.817022:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177037.817022:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177037.817026:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177037.817027:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177037.817033:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856328832:192.168.202.110@tcp:250:kworker.0
02000000:00000010:2.0:1786177037.817036:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:1.0:1786177037.817038:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1786177037.817041:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177037.817043:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177037.817047:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177037.817048:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786177037.817050:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786177037.817057:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000010:1.0:1786177037.817058:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
00000100:00000001:2.0:1786177037.817060:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177037.817063:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177037.817066:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786177037.817074:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177037.817082:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177037.817083:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00080000:2.0:1786177037.817088:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177037.817093:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:1.0:1786177037.817094:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000001:2.0:1786177037.817096:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177037.817100:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177037.817103:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177037.817107:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7261
00000100:00000200:1.0:1786177037.817109:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856328832, portal 25
00000100:00000001:1.0:1786177037.817117:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177037.817119:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000400:00000200:2.0:1786177037.817122:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1786177037.817130:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786177037.817130:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177037.817135:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177037.817144:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817150:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177037.817151:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177037.817156:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:1.0:1786177037.817160:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817161:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177037.817165:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000010:1.0:1786177037.817166:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000020:00000040:2.0:1786177037.817167:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177037.817170:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177037.817173:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000200:1.0:1786177037.817174:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856328832, offset 0
00000020:00000001:2.0:1786177037.817175:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786177037.817181:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786177037.817185:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817186:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786177037.817192:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177037.817192:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817197:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177037.817199:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786177037.817201:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817203:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177037.817204:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177037.817206:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786177037.817209:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177037.817210:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177037.817215:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177037.817217:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177037.817219:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177037.817221:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177037.817224:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000010:2.0:1786177037.817226:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786177037.817230:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177037.817232:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177037.817233:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177037.817234:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786177037.817236:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177037.817238:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177037.817240:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177037.817242:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000010:2.0:1786177037.817245:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000400:00000200:1.0:1786177037.817248:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1786177037.817251:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177037.817254:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00100000:2.0:1786177037.817259:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177037.817262:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254740:254740:256:4294967295] 192.168.202.110@tcp LPNI seq info [254729:254729:8:4294967295]
00000400:00000200:1.0:1786177037.817277:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177037.817280:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177037.817283:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177037.817289:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786177037.817293:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177037.817319:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817320:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177037.817325:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:1.0:1786177037.817327:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000100:00080000:2.0:1786177037.817328:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177037.817333:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177037.817334:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177037.817337:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.0:1786177037.817341:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786177037.817342:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7261
00000800:00000200:1.0:1786177037.817351:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000400:00000200:2.0:1786177037.817351:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177037.817355:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177037.817360:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177037.817365:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:1.1:1786177037.817366:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177037.817371:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1786177037.817374:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:2.0:1786177037.817376:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817382:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177037.817388:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177037.817389:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177037.817391:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177037.817393:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177037.817396:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000100:1.0:1786177037.817399:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000001:2.0:1786177037.817401:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000010:1.0:1786177037.817405:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000100:00000040:2.0:1786177037.817408:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817413:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786177037.817416:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1786177037.817417:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817421:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786177037.817427:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:2.0:1786177037.817427:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177037.817434:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:1.0:1786177037.817437:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000100:00000001:2.0:1786177037.817437:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177037.817441:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
02000000:00000001:2.0:1786177037.817442:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177037.817446:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177037.817449:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1786177037.817450:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177037.817453:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000200:1.0:1786177037.817457:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1786177037.817457:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177037.817463:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177037.817468:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177037.817475:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000100:00000001:1.0:1786177037.817476:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000040:2.0:1786177037.817481:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000040:1.0:1786177037.817482:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177037.817488:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177037.817492:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.817498:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177037.817498:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177037.817502:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786177037.817504:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1786177037.817506:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177037.817546:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786177037.817550:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.817556:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177037.817556:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177037.817566:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786177037.817567:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177037.817570:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.817573:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301937358
00000100:00000040:2.0:1786177037.817574:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177037.817584:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177037.817585:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1b40.
00000100:00080000:2.0:1786177037.817587:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177037.817592:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177037.817593:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177037.817595:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177037.817598:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177037.817598:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177037.817602:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7261
00000400:00000200:2.0:1786177037.817606:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177037.817608:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301937358
00000400:00000200:2.0:1786177037.817609:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177037.817615:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000010:1.0:1786177037.817617:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1d80.
00000100:00000040:2.0:1786177037.817618:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817624:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177037.817626:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:2.0:1786177037.817626:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817630:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177037.817634:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177037.817635:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:1.0:1786177037.817636:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000001:2.0:1786177037.817636:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177037.817640:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:1.0:1786177037.817642:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177037.817643:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786177037.817649:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786177037.817650:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1786177037.817652:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817655:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177037.817660:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00100000:1.0:1786177037.817662:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177037/real 1786177037]  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177037.817663:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177037.817667:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786177037.817674:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177037.817681:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177037.817687:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000001:1.0:1786177037.817690:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:2.0:1786177037.817691:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000001:1.0:1786177037.817694:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:2.0:1786177037.817696:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177037.817700:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:2.0:1786177037.817701:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177037.817705:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:1.0:1786177037.817707:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000008:00000001:2.0:1786177037.817710:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177037.817712:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:2.0:1786177037.817715:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.817718:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000010:2.0:1786177037.817722:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000100:00000200:1.0:1786177037.817725:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177037.817727:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177037.817734:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177037.817738:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177037.817743:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177037.817746:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177037.817751:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177037.817759:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1786177037.817759:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786177037.817766:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.817775:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00100000:2.0:1786177037.817777:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177037.817779:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.817785:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177037.817790:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177037.817792:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177037.817797:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177037.817801:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177037.817806:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.817826:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177037.817830:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177037.817837:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177037.817837:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7271)
00000100:00000001:1.0:1786177037.817850:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177037.817854:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.817856:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177037.817858:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.817861:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177037.817864:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177037.817867:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177037.817873:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177037.817876:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177037.817881:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177037.817891:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856328832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177037.817896:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.817899:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177037.817903:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856328832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.817917:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177037.817920:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177037.817923:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177037.817929:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177037.817931:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177037.817933:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786177037.817936:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177037.817939:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:1.0:1786177037.817944:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177037.817946:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177037.817948:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:1.0:1786177037.817952:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177037.817953:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.817956:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.817965:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177037.817966:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.817969:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177037.817970:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177037.817972:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177037.817976:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.817984:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177037.817986:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177037.817988:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177037.817990:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177037.817992:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177037.817996:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856328960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177037.818000:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177037.818002:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177037.818004:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177037.818006:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177037.818009:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177037.818011:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177037.818015:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177037.818019:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177037.818024:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177037.818027:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856328960, portal 10
00000100:00000001:1.0:1786177037.818029:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177037.818031:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:1.0:1786177037.818038:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818046:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177037.818048:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177037.818051:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177037.818054:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856328960, offset 0
00000400:00000200:1.0:1786177037.818059:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177037.818064:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177037.818075:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177037.818078:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177037.818082:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177037.818084:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177037.818087:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177037.818091:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254741:254741:256:4294967295] 192.168.202.110@tcp LPNI seq info [254730:254730:8:4294967295]
00000400:00000200:1.0:1786177037.818096:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177037.818103:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177037.818109:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177037.818113:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177037.818118:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177037.818121:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786177037.818126:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177037.818130:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1786177037.818134:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177037.818137:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177037.818140:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177037.818144:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177037.818147:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177037.818149:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177037.818153:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177037.818157:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818166:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177037.818169:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818177:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.818179:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177037.818182:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177037.818186:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786177037.818190:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177037.818192:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301933578
00000100:00000001:1.0:1786177037.818196:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301937358
00000100:00000001:1.0:1786177037.818198:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301933578
00000100:00000001:1.0:1786177037.818201:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301937358
00000100:00000001:1.0:1786177037.818203:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177037.818205:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.818208:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177037.818212:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177037/real 1786177037]  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177037.818222:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177037.818223:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177037.818225:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177037.818228:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177037.818230:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177037.818232:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177037.818236:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177037.818245:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818259:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177037.818261:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.818263:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177037.818265:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177037.818269:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818276:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177037.818280:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818288:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177037.818291:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.818293:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177037.818295:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.818297:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177037.818299:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177037.818319:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177037.818323:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177037.818326:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177037.818331:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177037.818340:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856328960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177037.818345:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177037.818349:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818356:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177037.818358:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177037.818360:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177037.818362:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177037.818365:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177037.818368:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177037.818372:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177037.818374:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177037.818376:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177037.818378:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177037.818382:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177037.818385:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177037.818388:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177037.818391:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177037.818395:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177037.818398:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856329088, portal 10
00000100:00000001:1.0:1786177037.818401:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177037.818402:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:1.0:1786177037.818408:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818415:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177037.818418:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177037.818421:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177037.818423:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856329088, offset 0
00000400:00000200:1.0:1786177037.818428:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177037.818433:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177037.818437:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177037.818440:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177037.818443:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177037.818445:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177037.818447:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177037.818450:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254742:254742:256:4294967295] 192.168.202.110@tcp LPNI seq info [254731:254731:8:4294967295]
00000400:00000200:1.0:1786177037.818454:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177037.818460:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177037.818472:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177037.818477:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177037.818481:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177037.818484:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786177037.818488:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177037.818491:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1786177037.818494:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177037.818497:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177037.818500:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177037.818503:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177037.818507:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177037.818508:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177037.818531:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177037.818535:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818544:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177037.818547:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818554:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.818556:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177037.818559:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177037.818563:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177037.818567:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177037.818570:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301933704
00000100:00000001:1.0:1786177037.818572:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301937358
00000100:00000001:1.0:1786177037.818575:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301933704
00000100:00000001:1.0:1786177037.818576:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301937358
00000100:00000001:1.0:1786177037.818579:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177037.818581:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.818584:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177037.818588:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177037/real 1786177037]  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177037.818596:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177037.818598:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177037.818600:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177037.818603:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177037.818604:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177037.818606:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177037.818609:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177037.818617:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818625:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177037.818627:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.818628:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177037.818630:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177037.818634:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818641:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177037.818645:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818652:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177037.818654:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.818656:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177037.818657:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.818659:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177037.818661:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177037.818664:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177037.818668:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177037.818671:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177037.818676:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177037.818684:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177037.818688:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177037.818691:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818698:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177037.818700:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177037.818702:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177037.818705:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177037.818707:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177037.818709:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177037.818712:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177037.818714:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177037.818716:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177037.818718:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177037.818722:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177037.818724:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177037.818728:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177037.818731:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177037.818734:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177037.818737:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856329216, portal 4
00000100:00000001:1.0:1786177037.818739:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177037.818741:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:1.0:1786177037.818747:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818754:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177037.818757:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177037.818759:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177037.818761:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856329216, offset 0
00000400:00000200:1.0:1786177037.818766:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177037.818770:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177037.818774:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177037.818777:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177037.818779:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177037.818782:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177037.818784:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177037.818787:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254743:254743:256:4294967295] 192.168.202.110@tcp LPNI seq info [254732:254732:8:4294967295]
00000400:00000200:1.0:1786177037.818792:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177037.818799:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177037.818804:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177037.818807:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177037.818811:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177037.818814:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786177037.818819:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177037.818822:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1786177037.818827:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177037.818829:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177037.818833:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177037.818836:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177037.818840:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177037.818842:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177037.818845:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177037.818849:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818857:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177037.818860:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818868:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.818869:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177037.818873:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177037.818876:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177037.818880:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177037.818883:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301933579
00000100:00000001:1.0:1786177037.818886:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301937358
00000100:00000001:1.0:1786177037.818888:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301933579
00000100:00000001:1.0:1786177037.818890:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301937358
00000100:00000001:1.0:1786177037.818893:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177037.818895:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.818897:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177037.818901:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177037/real 1786177037]  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177037.818908:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177037.818909:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177037.818910:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177037.818912:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177037.818914:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177037.818915:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177037.818918:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177037.818924:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818928:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177037.818928:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.818930:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177037.818931:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177037.818933:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818937:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177037.818940:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818944:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177037.818947:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.818948:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177037.818949:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.818950:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177037.818951:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177037.818953:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177037.818955:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177037.818956:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177037.818959:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177037.818963:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177037.818966:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177037.818967:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.818971:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177037.818972:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177037.818973:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177037.818975:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177037.818976:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177037.818978:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329344:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177037.818981:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177037.818982:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177037.818983:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177037.818984:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177037.818987:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177037.818989:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177037.818991:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177037.818992:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177037.818994:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177037.818997:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856329344, portal 4
00000100:00000001:1.0:1786177037.818998:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177037.818999:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00000040:1.0:1786177037.819004:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.819008:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177037.819010:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177037.819012:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177037.819014:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856329344, offset 0
00000400:00000200:1.0:1786177037.819018:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177037.819022:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177037.819027:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177037.819030:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177037.819032:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177037.819035:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177037.819037:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177037.819040:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254744:254744:256:4294967295] 192.168.202.110@tcp LPNI seq info [254733:254733:8:4294967295]
00000400:00000200:1.0:1786177037.819045:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177037.819051:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177037.819056:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177037.819060:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177037.819064:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177037.819068:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1786177037.819072:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177037.819075:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1786177037.819079:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177037.819081:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177037.819084:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177037.819088:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177037.819091:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177037.819093:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177037.819096:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177037.819099:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.819116:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177037.819119:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.819127:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.819129:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177037.819133:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177037.819137:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177037.819140:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177037.819142:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301933587
00000100:00000001:1.0:1786177037.819144:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301937358
00000100:00000001:1.0:1786177037.819146:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301933587
00000100:00000001:1.0:1786177037.819149:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301937358
00000100:00000001:1.0:1786177037.819151:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177037.819154:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.819156:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177037.819161:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177037/real 1786177037]  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177037.819170:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177037.819172:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177037.819174:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177037.819177:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177037.819179:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177037.819182:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177037.819185:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177037.819195:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.819203:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177037.819206:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.819208:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177037.819210:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177037.819214:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.819222:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177037.819227:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.819235:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177037.819238:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.819240:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177037.819241:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.819243:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177037.819245:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177037.819248:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177037.819251:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177037.819253:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177037.819257:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177037.819265:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329344:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177037.819268:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.819271:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177037.819274:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856328960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.819280:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177037.819282:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177037.819284:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177037.819287:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177037.819289:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177037.819291:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177037.819294:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177037.819296:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:1.0:1786177037.819299:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177037.819316:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177037.819319:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:1.0:1786177037.819322:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177037.819324:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.819326:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177037.819330:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856329088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.819338:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177037.819339:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177037.819340:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177037.819342:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177037.819343:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177037.819344:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177037.819345:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177037.819347:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:1.0:1786177037.819350:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177037.819351:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177037.819352:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:1.0:1786177037.819353:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177037.819354:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.819356:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177037.819358:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856329216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.819362:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177037.819363:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177037.819364:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177037.819366:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177037.819367:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177037.819368:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177037.819369:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177037.819370:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:1.0:1786177037.819371:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177037.819373:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177037.819374:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:1.0:1786177037.819375:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177037.819376:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.819377:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177037.819379:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856329344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177037.819385:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177037.819386:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177037.819388:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177037.819390:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177037.819391:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177037.819392:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177037.819394:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177037.819396:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:1.0:1786177037.819398:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177037.819399:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177037.819401:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:1.0:1786177037.819403:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177037.819404:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.819406:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177037.819411:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177037.819412:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177037.819415:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177037.819417:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177038.392663:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177038.392671:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177038.392675:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177038.392676:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177038.392679:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177038.392680:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177038.392753:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177038.392758:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177038.392762:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177038.840692:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177038.840701:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177038.840703:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177038.840709:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177038.840713:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177038.840718:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177038.840721:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177038.840725:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177038.840728:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177039.416687:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177039.416698:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177039.416704:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177039.416706:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177039.416709:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177039.416711:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177039.416770:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177039.416775:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177039.416779:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177039.864665:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177039.864677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177039.864680:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177039.864687:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177039.864692:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177039.864697:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177039.864701:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177039.864705:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177039.864708:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177040.440658:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177040.440666:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177040.440671:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177040.440672:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177040.440674:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177040.440675:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177040.440741:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177040.440746:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177040.440751:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177040.888617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177040.888625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177040.888627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177040.888633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177040.888636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177040.888640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177040.888642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177040.888646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177040.888648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177041.464633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177041.464640:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177041.464642:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177041.464645:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177041.464648:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177041.464650:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177041.464652:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177041.464653:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177041.464656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177041.464657:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177041.464658:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177041.464659:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177041.464660:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177041.464661:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177041.464662:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177041.464664:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177041.464666:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177041.464667:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177041.464735:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177041.464740:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177041.464743:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177041.656611:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301937358, at 4301937408
00000100:00000040:1.0:1786177041.656614:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301937358, at 4301937408
00000100:00000010:0.0:1786177041.656626:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1b40.
00000100:00000010:1.0:1786177041.656628:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1d80.
00000100:00000001:0.0:1786177041.912638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177041.912645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177041.912647:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177041.912651:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177041.912654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177041.912657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177041.912659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177041.912662:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177041.912664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177042.488620:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177042.488626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177042.488631:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177042.488632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177042.488635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177042.488636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177042.488720:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177042.488727:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177042.488732:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177042.936574:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177042.936583:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177042.936585:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177042.936592:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177042.936596:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177042.936600:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177042.936602:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177042.936603:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177042.936608:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177042.936610:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177042.936612:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.936614:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177042.936618:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177042.936623:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177042.936631:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7266
00000400:00000200:2.0:1786177042.936640:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177042.936648:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177042.936651:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177042.936655:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.936659:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177042.936661:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.936663:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177042.936667:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177042.936668:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177042.936671:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177042.936674:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177042.936676:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177042.936678:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177042.936680:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.936682:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177042.936683:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.936684:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177042.936687:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177042.936689:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177042.936691:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177042.936693:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177042.936696:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177042.936698:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.936700:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177042.936702:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177042.936712:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000020:00000040:2.0:1786177042.936715:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177042.936719:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177042.936720:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.936723:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177042.936726:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177042.936731:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177042.936739:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786177042.936746:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177042.936756:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177042.936766:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177042.936771:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177042.936775:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177042.936791:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177042.936794:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177042.936797:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177042.936798:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177042.936799:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177042.936801:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177042.936803:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7266
00000400:00000200:2.0:1786177042.936805:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177042.936807:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177042.936809:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177042.936811:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.936813:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177042.936815:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.936816:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177042.936818:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177042.936819:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177042.936820:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177042.936821:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177042.936823:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177042.936825:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177042.936829:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.936831:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177042.936835:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.936838:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177042.936841:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1786177042.936841:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786177042.936848:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1786177042.936849:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177042.936853:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786177042.936855:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177042.936859:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1786177042.936865:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177042.936867:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177042.936873:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.936879:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177042.936885:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177042.936893:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.936895:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177042.936898:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.936899:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177042.936908:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000010:2.0:1786177042.936908:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
02000000:00000001:1.0:1786177042.936913:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1786177042.936915:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786177042.936919:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177042.936920:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177042.936923:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.936926:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:1.0:1786177042.936929:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856329472:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1786177042.936929:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177042.936932:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177042.936938:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:1.0:1786177042.936939:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1786177042.936943:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177042.936947:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:2.0:1786177042.936951:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1786177042.936953:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177042.936960:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786177042.936965:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177042.936968:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177042.936972:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b02c00.
00000100:00000040:2.0:1786177042.936972:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786177042.936978:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.936986:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177042.936990:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1786177042.936991:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786177042.936998:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:1.0:1786177042.937001:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:2.0:1786177042.937001:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177042.937004:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177042.937008:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177042.937010:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7266
00000400:00000010:1.0:1786177042.937013:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000400:00000200:2.0:1786177042.937013:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177042.937019:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177042.937021:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177042.937023:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:1.0:1786177042.937026:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856329472, portal 25
00000100:00000001:2.0:1786177042.937026:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177042.937030:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177042.937034:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177042.937034:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177042.937039:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786177042.937040:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454188800 : -114548255362816 : ffff97d1a7edf100)
00000020:00000040:2.0:1786177042.937040:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177042.937043:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177042.937045:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177042.937048:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786177042.937050:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:1.0:1786177042.937054:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177042.937055:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.937058:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177042.937062:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.937065:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177042.937070:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786177042.937075:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177042.937079:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786177042.937080:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177042.937083:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177042.937086:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1786177042.937090:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:2.0:1786177042.937091:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.937095:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177042.937097:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1786177042.937098:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000008:00000001:2.0:1786177042.937099:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.937102:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177042.937107:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856329472, offset 0
00000100:00000010:2.0:1786177042.937108:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000020:00000040:2.0:1786177042.937112:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177042.937114:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177042.937115:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.937119:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1786177042.937120:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:2.0:1786177042.937122:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177042.937125:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177042.937128:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786177042.937130:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177042.937134:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1786177042.937138:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177042.937145:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177042.937148:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177042.937149:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1786177042.937153:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177042.937160:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177042.937161:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1786177042.937162:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177042.937167:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177042.937168:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177042.937169:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786177042.937172:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1786177042.937174:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177042.937178:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7266
00000400:00000200:2.0:1786177042.937182:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177042.937184:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177042.937187:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177042.937190:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1786177042.937191:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:2.0:1786177042.937194:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.937197:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786177042.937202:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254745:254745:256:4294967295] 192.168.202.110@tcp LPNI seq info [254734:254734:8:4294967295]
00000100:00000040:2.0:1786177042.937202:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.937207:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177042.937212:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:1.0:1786177042.937215:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786177042.937216:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177042.937219:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177042.937221:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177042.937223:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177042.937228:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:1.0:1786177042.937233:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1786177042.937235:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.937240:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177042.937242:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.937246:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:1.0:1786177042.937247:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:2.0:1786177042.937249:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177042.937266:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177042.937268:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786177042.937269:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
02000000:00000001:2.0:1786177042.937271:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177042.937274:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177042.937276:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.937279:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000800:00000200:1.0:1786177042.937280:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000008:00100000:2.0:1786177042.937282:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000800:00000200:1.0:1786177042.937285:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000008:00000001:2.0:1786177042.937286:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.937289:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177042.937293:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000800:00000200:1.1:1786177042.937294:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1786177042.937297:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000800:00000200:1.1:1786177042.937299:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
02000000:00000001:2.0:1786177042.937301:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177042.937304:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1786177042.937309:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177042.937309:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177042.937313:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000010:1.0:1786177042.937314:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
02000000:00000001:2.0:1786177042.937315:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177042.937322:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000400:00000200:1.0:1786177042.937325:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1786177042.937325:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177042.937332:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177042.937334:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1786177042.937342:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177042.937346:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177042.937350:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177042.937353:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000040:2.0:1786177042.937357:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937359:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177042.937365:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177042.937372:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177042.937377:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177042.937382:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177042.937384:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1786177042.937385:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:2.0:1786177042.937386:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177042.937391:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:1.0:1786177042.937394:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786177042.937395:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7266
00000400:00000200:2.0:1786177042.937402:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177042.937406:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177042.937410:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177042.937415:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.937419:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177042.937421:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177042.937422:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177042.937427:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177042.937427:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177042.937432:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177042.937433:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177042.937434:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177042.937436:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786177042.937438:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786177042.937439:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177042.937440:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177042.937447:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177042.937449:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786177042.937450:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786177042.937454:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177042.937456:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301942477
00000100:00000001:2.0:1786177042.937457:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177042.937462:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786177042.937464:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000010:1.0:1786177042.937466:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1d80.
00000100:00000001:2.0:1786177042.937467:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177042.937471:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177042.937474:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177042.937475:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.937477:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177042.937478:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000008:00100000:2.0:1786177042.937481:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177042.937485:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.937486:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301942477
00000100:00000001:2.0:1786177042.937487:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177042.937494:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1d20.
00000100:00000010:2.0:1786177042.937496:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000100:00000001:1.0:1786177042.937499:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000020:00000040:2.0:1786177042.937500:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:1.0:1786177042.937504:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177042.937504:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177042.937507:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177042.937534:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786177042.937535:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177042.937536:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177042.937539:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.937542:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000010:2.0:1786177042.937544:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786177042.937548:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177042.937551:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177042/real 1786177042]  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177042.937555:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177042.937584:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000001:1.0:1786177042.937585:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786177042.937586:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786177042.937589:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:2.0:1786177042.937592:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1786177042.937594:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177042.937600:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177042.937603:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:2.0:1786177042.937605:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.937607:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:2.0:1786177042.937611:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7276)
00000100:00000200:1.0:1786177042.937613:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177042.937621:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937627:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177042.937628:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.937630:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177042.937632:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177042.937635:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937640:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177042.937643:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937649:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177042.937651:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.937652:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177042.937652:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.937654:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177042.937656:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177042.937658:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177042.937661:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177042.937663:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177042.937667:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177042.937673:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856329472:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177042.937676:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.937677:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177042.937679:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf100 x1872934856329472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937684:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177042.937686:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177042.937688:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177042.937691:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177042.937692:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177042.937693:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786177042.937695:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177042.937697:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:1.0:1786177042.937700:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177042.937702:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177042.937703:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf100.
00000100:00000001:1.0:1786177042.937705:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177042.937706:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.937708:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.937715:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177042.937716:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.937718:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177042.937719:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177042.937720:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177042.937723:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937728:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177042.937729:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177042.937731:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177042.937732:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177042.937734:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177042.937736:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329600:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177042.937739:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177042.937741:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177042.937742:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177042.937743:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177042.937746:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177042.937747:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177042.937752:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177042.937755:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177042.937758:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177042.937762:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856329600, portal 10
00000100:00000001:1.0:1786177042.937764:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177042.937766:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000100:00000040:1.0:1786177042.937773:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937780:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177042.937782:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177042.937784:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177042.937786:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856329600, offset 0
00000400:00000200:1.0:1786177042.937789:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177042.937792:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177042.937795:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177042.937798:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177042.937800:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177042.937802:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177042.937804:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177042.937806:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254746:254746:256:4294967295] 192.168.202.110@tcp LPNI seq info [254735:254735:8:4294967295]
00000400:00000200:1.0:1786177042.937809:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177042.937814:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177042.937817:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177042.937820:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177042.937823:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177042.937825:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786177042.937828:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177042.937830:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1786177042.937832:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177042.937834:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177042.937836:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177042.937838:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177042.937841:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177042.937842:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177042.937844:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177042.937846:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937852:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177042.937854:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937859:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.937860:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177042.937862:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177042.937865:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786177042.937867:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177042.937869:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301938698
00000100:00000001:1.0:1786177042.937871:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301942477
00000100:00000001:1.0:1786177042.937873:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301938698
00000100:00000001:1.0:1786177042.937875:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301942477
00000100:00000001:1.0:1786177042.937876:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177042.937877:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.937878:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177042.937881:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177042/real 1786177042]  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177042.937890:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177042.937891:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177042.937893:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177042.937894:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177042.937895:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177042.937896:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177042.937898:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177042.937904:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937909:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177042.937910:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.937911:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177042.937912:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177042.937914:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937918:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177042.937920:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937924:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177042.937926:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.937927:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177042.937927:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.937929:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177042.937930:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177042.937931:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177042.937933:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177042.937935:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177042.937938:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177042.937943:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329600:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177042.937946:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177042.937948:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937952:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177042.937953:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177042.937954:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177042.937955:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177042.937957:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177042.937958:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177042.937960:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177042.937961:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177042.937962:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177042.937963:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177042.937966:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177042.937967:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177042.937970:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177042.937971:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177042.937973:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177042.937975:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856329728, portal 10
00000100:00000001:1.0:1786177042.937976:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177042.937977:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178944 : -114548255372672 : ffff97d1a7edca80)
00000100:00000040:1.0:1786177042.937982:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.937986:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177042.937988:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177042.937989:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177042.937991:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856329728, offset 0
00000400:00000200:1.0:1786177042.937995:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177042.937997:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177042.937999:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177042.938001:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177042.938003:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177042.938005:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177042.938006:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177042.938008:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254747:254747:256:4294967295] 192.168.202.110@tcp LPNI seq info [254736:254736:8:4294967295]
00000400:00000200:1.0:1786177042.938011:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177042.938015:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177042.938018:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177042.938019:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177042.938021:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177042.938023:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786177042.938026:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177042.938027:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1786177042.938030:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177042.938031:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177042.938033:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177042.938035:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177042.938038:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177042.938039:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177042.938040:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177042.938043:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938048:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177042.938052:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938059:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.938060:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177042.938064:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177042.938067:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177042.938070:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177042.938073:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301938823
00000100:00000001:1.0:1786177042.938075:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301942477
00000100:00000001:1.0:1786177042.938078:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301938823
00000100:00000001:1.0:1786177042.938080:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301942477
00000100:00000001:1.0:1786177042.938082:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177042.938085:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.938087:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177042.938092:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177042/real 1786177042]  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177042.938101:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177042.938102:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177042.938105:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177042.938107:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177042.938108:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177042.938109:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177042.938112:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177042.938118:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938123:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177042.938124:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938125:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177042.938127:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177042.938129:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938135:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177042.938139:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938146:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177042.938150:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938152:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177042.938153:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938155:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177042.938157:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177042.938160:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177042.938164:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177042.938166:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177042.938171:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177042.938180:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177042.938184:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177042.938186:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938192:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177042.938193:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177042.938194:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177042.938196:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177042.938197:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177042.938199:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177042.938201:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177042.938202:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177042.938204:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177042.938205:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177042.938208:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177042.938209:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177042.938212:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177042.938214:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177042.938216:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177042.938219:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856329856, portal 4
00000100:00000001:1.0:1786177042.938221:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177042.938222:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000040:1.0:1786177042.938226:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938231:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177042.938233:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177042.938234:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177042.938236:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856329856, offset 0
00000400:00000200:1.0:1786177042.938238:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177042.938241:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177042.938243:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177042.938245:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177042.938247:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177042.938249:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177042.938250:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177042.938270:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254748:254748:256:4294967295] 192.168.202.110@tcp LPNI seq info [254737:254737:8:4294967295]
00000400:00000200:1.0:1786177042.938274:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177042.938279:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177042.938282:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177042.938285:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177042.938288:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177042.938290:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786177042.938293:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177042.938295:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1786177042.938297:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177042.938299:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177042.938301:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177042.938304:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177042.938306:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177042.938307:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177042.938309:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177042.938311:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938321:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177042.938324:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938329:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.938330:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177042.938332:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177042.938334:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177042.938337:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177042.938338:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301938698
00000100:00000001:1.0:1786177042.938340:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301942477
00000100:00000001:1.0:1786177042.938342:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301938698
00000100:00000001:1.0:1786177042.938343:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301942477
00000100:00000001:1.0:1786177042.938345:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177042.938346:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.938347:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177042.938349:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177042/real 1786177042]  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177042.938354:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177042.938355:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177042.938356:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177042.938357:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177042.938358:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177042.938360:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177042.938362:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177042.938368:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938372:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177042.938373:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938374:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177042.938375:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177042.938378:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938383:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177042.938387:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938395:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177042.938397:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938399:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177042.938400:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938402:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177042.938404:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177042.938407:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177042.938411:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177042.938413:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177042.938418:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177042.938427:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177042.938432:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177042.938435:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938441:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177042.938442:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177042.938444:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177042.938446:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177042.938447:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177042.938450:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177042.938452:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177042.938453:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177042.938455:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177042.938456:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177042.938459:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177042.938460:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177042.938463:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177042.938464:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177042.938466:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177042.938469:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856329984, portal 4
00000100:00000001:1.0:1786177042.938470:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177042.938471:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00000040:1.0:1786177042.938476:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938480:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177042.938482:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177042.938483:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177042.938485:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856329984, offset 0
00000400:00000200:1.0:1786177042.938487:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177042.938489:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177042.938492:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177042.938494:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177042.938496:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177042.938497:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177042.938499:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177042.938501:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254749:254749:256:4294967295] 192.168.202.110@tcp LPNI seq info [254738:254738:8:4294967295]
00000400:00000200:1.0:1786177042.938504:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177042.938508:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177042.938526:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177042.938529:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177042.938532:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177042.938534:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1786177042.938537:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177042.938539:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1786177042.938541:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177042.938543:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177042.938545:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177042.938547:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177042.938550:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177042.938551:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177042.938553:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177042.938555:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938561:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177042.938563:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938568:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.938569:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177042.938571:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177042.938573:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177042.938575:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177042.938577:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301938707
00000100:00000001:1.0:1786177042.938579:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301942477
00000100:00000001:1.0:1786177042.938580:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301938707
00000100:00000001:1.0:1786177042.938582:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301942477
00000100:00000001:1.0:1786177042.938583:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177042.938584:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.938585:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177042.938588:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177042/real 1786177042]  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177042.938593:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177042.938594:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177042.938595:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177042.938597:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177042.938598:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177042.938599:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177042.938601:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177042.938606:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938610:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177042.938611:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938612:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177042.938613:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177042.938615:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938619:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177042.938621:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938626:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177042.938627:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938629:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177042.938630:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938631:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177042.938632:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177042.938633:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177042.938635:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177042.938636:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177042.938639:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177042.938648:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856329984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177042.938651:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938652:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177042.938654:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856329600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938658:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177042.938659:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177042.938661:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177042.938663:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177042.938664:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177042.938665:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177042.938667:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177042.938668:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0800.
02000000:00000001:1.0:1786177042.938671:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177042.938672:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177042.938673:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
00000100:00000001:1.0:1786177042.938675:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177042.938676:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938678:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177042.938680:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edca80 x1872934856329728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938684:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177042.938684:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177042.938685:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177042.938687:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177042.938687:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177042.938688:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177042.938689:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177042.938690:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:1.0:1786177042.938691:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177042.938692:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177042.938693:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edca80.
00000100:00000001:1.0:1786177042.938694:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177042.938695:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938697:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177042.938699:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856329856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938703:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177042.938704:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177042.938704:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177042.938707:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177042.938708:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177042.938708:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177042.938709:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177042.938710:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:1.0:1786177042.938711:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177042.938712:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177042.938713:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:1.0:1786177042.938714:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177042.938715:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938716:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177042.938717:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856329984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177042.938721:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177042.938722:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177042.938723:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177042.938724:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177042.938724:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177042.938725:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177042.938726:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177042.938727:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:1.0:1786177042.938728:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177042.938729:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177042.938730:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:1.0:1786177042.938731:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177042.938731:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938733:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177042.938735:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177042.938736:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177042.938738:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177042.938739:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177043.512685:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177043.512696:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177043.512703:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177043.512705:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177043.512709:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177043.512714:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177043.512717:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177043.512722:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177043.512726:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177043.512729:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177043.512731:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177043.512734:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177043.512814:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177043.512819:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177043.512823:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177043.960664:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177043.960674:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177043.960676:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177043.960682:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177043.960685:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177043.960689:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177043.960692:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177043.960695:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177043.960697:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177044.536637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177044.536643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177044.536647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177044.536648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177044.536650:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177044.536651:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177044.536729:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177044.536737:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177044.536742:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177044.984603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177044.984615:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177044.984618:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177044.984627:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177044.984632:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177044.984636:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177044.984639:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177044.984643:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177044.984648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177045.560619:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177045.560624:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177045.560628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177045.560629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177045.560632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177045.560632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177045.560726:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177045.560734:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177045.560739:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177046.008628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177046.008633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177046.008635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177046.008640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177046.008643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177046.008646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177046.008648:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177046.008650:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177046.008652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177046.584596:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177046.584603:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177046.584610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177046.584611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177046.584615:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177046.584617:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177046.584674:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177046.584677:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177046.584681:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177046.776624:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301942477, at 4301942528
00000100:00000040:1.0:1786177046.776624:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301942477, at 4301942528
00000100:00000010:0.0:1786177046.776640:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1d80.
00000100:00000010:1.0:1786177046.776653:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1d20.
00000100:00000001:0.0:1786177047.032631:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177047.032638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177047.032641:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177047.032646:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177047.032650:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177047.032654:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177047.032657:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177047.032661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177047.032664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177047.608620:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177047.608627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177047.608633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177047.608635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177047.608638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177047.608639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177047.608696:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177047.608700:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177047.608703:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177048.056619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177048.056625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177048.056635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177048.056641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:3.0:1786177048.056644:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177048.056645:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177048.056649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177048.056653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1786177048.056653:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177048.056657:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177048.056659:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177048.056661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177048.056664:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177048.056670:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177048.056678:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7271
00000400:00000200:3.0:1786177048.056688:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177048.056696:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177048.056700:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177048.056703:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.056707:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177048.056710:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.056712:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177048.056716:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177048.056719:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177048.056722:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177048.056725:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177048.056728:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177048.056731:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177048.056734:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.056736:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177048.056738:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.056741:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177048.056744:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177048.056748:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177048.056750:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177048.056754:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177048.056757:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177048.056761:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177048.056764:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177048.056766:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177048.056775:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d180.
00000020:00000040:3.0:1786177048.056780:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177048.056784:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177048.056786:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177048.056789:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177048.056792:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177048.056796:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177048.056804:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786177048.056812:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177048.056821:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177048.056831:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177048.056836:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177048.056840:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085df314e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177048.056864:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177048.056867:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177048.056880:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177048.056883:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177048.056884:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177048.056886:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177048.056888:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786177048.056891:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177048.056895:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:3.0:1786177048.056895:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7271
00000400:00000200:3.0:1786177048.056904:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1786177048.056905:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177048.056911:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786177048.056918:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1786177048.056918:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786177048.056921:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177048.056925:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:1.0:1786177048.056926:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177048.056930:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177048.056930:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1786177048.056933:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177048.056937:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00100000:1.0:1786177048.056939:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856330112:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177048.056943:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177048.056945:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:3.0:1786177048.056949:0:62275:0:(genops.c:813:class_conn2export()) Process entered
02000000:00000001:1.0:1786177048.056950:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:3.0:1786177048.056953:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177048.056956:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177048.056958:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
02000000:00000001:1.0:1786177048.056962:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177048.056963:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:1.0:1786177048.056969:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:3.0:1786177048.056969:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177048.056977:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000010:1.0:1786177048.056978:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
00000100:00000001:3.0:1786177048.056980:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1786177048.056983:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177048.056987:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.056991:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786177048.056995:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:3.0:1786177048.056997:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000400:00000010:1.0:1786177048.057002:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00080000:3.0:1786177048.057003:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057011:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177048.057012:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
02000000:00000001:3.0:1786177048.057014:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:1.0:1786177048.057022:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856330112, portal 25
02000000:00000001:3.0:1786177048.057023:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177048.057027:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
02000000:00000001:3.0:1786177048.057029:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.057030:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297088 : -114547945254528 : ffff97d1ba69d180)
00000100:00000001:3.0:1786177048.057035:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:1.0:1786177048.057039:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1786177048.057042:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177048.057049:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.057051:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786177048.057053:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177048.057056:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177048.057061:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177048.057063:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856330112, offset 0
00000100:00000010:3.0:1786177048.057064:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e300.
00000020:00000040:3.0:1786177048.057068:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000400:00000200:1.0:1786177048.057071:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786177048.057074:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177048.057077:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177048.057080:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1786177048.057083:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786177048.057083:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1786177048.057094:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:3.0:1786177048.057094:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177048.057103:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000400:00000200:1.0:1786177048.057104:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177048.057109:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177048.057113:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177048.057118:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057122:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177048.057125:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254750:254750:256:4294967295] 192.168.202.110@tcp LPNI seq info [254739:254739:8:4294967295]
00000100:00100000:3.0:1786177048.057133:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177048.057135:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177048.057146:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177048.057152:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177048.057155:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1786177048.057161:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e3752a60] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1786177048.057167:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177048.057171:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177048.057182:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177048.057185:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786177048.057186:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00080000:3.0:1786177048.057191:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.1:1786177048.057194:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786177048.057201:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000800:00000200:1.1:1786177048.057201:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786177048.057221:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177048.057223:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177048.057229:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177048.057232:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1786177048.057237:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1786177048.057239:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000100:00080000:3.0:1786177048.057241:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1786177048.057242:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177048.057248:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177048.057252:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177048.057256:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1786177048.057263:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1786177048.057264:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7271
00000100:00000040:1.0:1786177048.057269:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177048.057275:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177048.057285:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786177048.057286:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177048.057289:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786177048.057290:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:3.0:1786177048.057297:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177048.057300:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177048.057301:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177048.057307:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177048.057310:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177048.057313:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177048.057316:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301947597
00000020:00000001:3.0:1786177048.057318:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177048.057322:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177048.057324:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000010:1.0:1786177048.057325:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1d20.
00000020:00000040:3.0:1786177048.057328:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786177048.057332:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:1.0:1786177048.057334:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000001:3.0:1786177048.057337:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:1.0:1786177048.057340:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301947597
00000100:00000040:3.0:1786177048.057344:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000010:1.0:1786177048.057345:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1720.
00000100:00000001:3.0:1786177048.057350:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177048.057351:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:3.0:1786177048.057356:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177048.057357:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177048.057363:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177048.057364:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.057368:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000020:00000040:3.0:1786177048.057369:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00100000:1.0:1786177048.057374:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177048/real 1786177048]  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177048.057382:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177048.057385:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:3.0:1786177048.057385:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:1.0:1786177048.057388:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:3.0:1786177048.057391:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177048.057393:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177048.057397:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
02000000:00000001:3.0:1786177048.057397:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786177048.057402:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:3.0:1786177048.057404:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1786177048.057406:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177048.057410:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177048.057416:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000200:1.0:1786177048.057419:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1786177048.057421:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177048.057428:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177048.057434:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.057435:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177048.057438:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.057441:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177048.057442:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:3.0:1786177048.057442:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c700.
00000020:00000040:3.0:1786177048.057446:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00100000:1.0:1786177048.057449:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177048.057451:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177048.057454:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177048.057458:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786177048.057461:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177048.057461:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177048.057465:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177048.057468:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1786177048.057472:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:1.0:1786177048.057477:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177048.057478:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.057483:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.057486:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177048.057488:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177048.057488:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177048.057493:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177048.057495:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177048.057501:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:3.0:1786177048.057502:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177048.057506:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177048.057507:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000040:3.0:1786177048.057536:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e56458ae] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.057546:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1786177048.057547:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177048.057551:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177048.057554:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177048.057557:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:1.0:1786177048.057558:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786177048.057559:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177048.057565:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177048.057569:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7271
00000400:00000200:3.0:1786177048.057576:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177048.057580:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00100000:1.0:1786177048.057581:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856330112:192.168.202.110@tcp:250:kworker.0
00000100:00080000:3.0:1786177048.057582:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177048.057588:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057593:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177048.057596:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177048.057596:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057602:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177048.057603:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000001:3.0:1786177048.057604:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177048.057607:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177048.057609:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000040:1.0:1786177048.057611:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d180 x1872934856330112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177048.057612:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177048.057615:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177048.057617:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177048.057620:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057621:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177048.057622:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057623:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177048.057625:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177048.057629:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057635:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177048.057639:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786177048.057642:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177048.057644:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177048.057647:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000001:3.0:1786177048.057649:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177048.057653:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
00000100:00000001:3.0:1786177048.057653:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177048.057658:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000001:1.0:1786177048.057660:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000008:00000001:3.0:1786177048.057663:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1786177048.057664:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177048.057668:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057668:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1786177048.057673:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177048.057677:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
00000100:00000010:3.0:1786177048.057677:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c000.
02000000:00000001:1.0:1786177048.057682:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000020:00000040:3.0:1786177048.057684:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:1.0:1786177048.057685:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177048.057687:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d180.
02000000:00000001:3.0:1786177048.057690:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786177048.057691:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
02000000:00000001:3.0:1786177048.057693:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.057694:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177048.057697:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1786177048.057698:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177048.057700:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177048.057703:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.057706:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000010:3.0:1786177048.057707:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:1.0:1786177048.057708:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177048.057711:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.057712:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177048.057715:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177048.057717:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00100000:3.0:1786177048.057719:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1786177048.057724:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177048.057741:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177048.057742:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177048.057745:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177048.057747:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177048.057748:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177048.057751:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177048.057754:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177048.057758:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006fe8fdb7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177048.057759:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856330240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177048.057765:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177048.057768:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177048.057770:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177048.057774:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177048.057780:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
00000100:00000001:3.0:1786177048.057781:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177048.057786:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:1.0:1786177048.057787:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177048.057790:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177048.057793:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177048.057795:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786177048.057796:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177048.057798:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177048.057801:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7271
00000400:00000200:3.0:1786177048.057804:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177048.057806:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177048.057808:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177048.057811:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:1.0:1786177048.057812:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:3.0:1786177048.057813:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177048.057817:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057820:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000010:1.0:1786177048.057821:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000020:00000001:3.0:1786177048.057823:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177048.057826:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000200:1.0:1786177048.057829:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856330240, portal 10
00000020:00000001:3.0:1786177048.057829:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177048.057831:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:1.0:1786177048.057835:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000020:00000001:3.0:1786177048.057836:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786177048.057841:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000020:00000001:3.0:1786177048.057841:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177048.057847:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057849:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177048.057851:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057852:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177048.057855:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000040:1.0:1786177048.057857:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786177048.057859:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057864:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177048.057867:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177048.057870:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177048.057874:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.057877:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786177048.057878:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177048.057883:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177048.057886:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177048.057887:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057888:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177048.057894:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000010:3.0:1786177048.057895:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ea00.
00000020:00000040:3.0:1786177048.057900:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000200:1.0:1786177048.057901:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856330240, offset 0
02000000:00000001:3.0:1786177048.057902:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177048.057905:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177048.057908:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177048.057909:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1786177048.057912:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786177048.057912:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177048.057918:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000400:00000200:1.0:1786177048.057923:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177048.057925:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177048.057931:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:3.0:1786177048.057932:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177048.057941:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1786177048.057942:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177048.057946:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177048.057947:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:3.0:1786177048.057952:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053162887] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786177048.057955:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177048.057963:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1786177048.057965:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177048.057970:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254751:254751:256:4294967295] 192.168.202.110@tcp LPNI seq info [254740:254740:8:4294967295]
00000100:00000040:3.0:1786177048.057970:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7281)
00000400:00000200:1.0:1786177048.057978:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177048.057986:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177048.057992:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177048.057996:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177048.058000:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177048.058003:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786177048.058007:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177048.058010:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786177048.058014:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177048.058016:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177048.058019:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177048.058022:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177048.058025:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177048.058027:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177048.058030:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177048.058034:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058042:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177048.058044:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058050:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.058052:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177048.058055:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177048.058059:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786177048.058062:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177048.058064:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301943818
00000100:00000001:1.0:1786177048.058066:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301947597
00000100:00000001:1.0:1786177048.058069:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301943818
00000100:00000001:1.0:1786177048.058071:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301947597
00000100:00000001:1.0:1786177048.058073:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177048.058075:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.058077:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177048.058081:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177048/real 1786177048]  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177048.058089:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177048.058091:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177048.058092:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177048.058095:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177048.058097:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177048.058099:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177048.058102:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177048.058110:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058117:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177048.058118:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058120:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177048.058121:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177048.058126:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058133:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177048.058137:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058144:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177048.058147:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058148:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177048.058149:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058151:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177048.058153:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177048.058156:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177048.058160:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177048.058162:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177048.058165:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177048.058173:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856330240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177048.058177:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177048.058181:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058188:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177048.058190:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177048.058192:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177048.058194:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177048.058196:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177048.058199:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856330368:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177048.058217:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177048.058219:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177048.058221:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177048.058223:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177048.058227:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177048.058229:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177048.058233:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177048.058235:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177048.058238:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177048.058242:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856330368, portal 10
00000100:00000001:1.0:1786177048.058244:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177048.058245:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:1.0:1786177048.058251:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058265:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177048.058268:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177048.058269:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177048.058271:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856330368, offset 0
00000400:00000200:1.0:1786177048.058275:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177048.058279:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177048.058283:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177048.058286:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177048.058288:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177048.058291:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177048.058293:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177048.058296:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254752:254752:256:4294967295] 192.168.202.110@tcp LPNI seq info [254741:254741:8:4294967295]
00000400:00000200:1.0:1786177048.058301:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177048.058308:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177048.058313:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177048.058317:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177048.058322:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177048.058325:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786177048.058329:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177048.058332:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786177048.058335:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177048.058336:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177048.058339:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177048.058341:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177048.058344:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177048.058345:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177048.058347:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177048.058349:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058355:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177048.058357:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058366:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.058367:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177048.058369:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177048.058371:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177048.058374:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177048.058375:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301943943
00000100:00000001:1.0:1786177048.058377:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301947597
00000100:00000001:1.0:1786177048.058378:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301943943
00000100:00000001:1.0:1786177048.058380:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301947597
00000100:00000001:1.0:1786177048.058381:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177048.058382:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.058384:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177048.058386:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177048/real 1786177048]  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177048.058391:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177048.058392:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177048.058393:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177048.058394:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177048.058396:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177048.058397:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177048.058399:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177048.058404:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058408:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177048.058409:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058410:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177048.058411:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177048.058413:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058418:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177048.058421:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058425:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177048.058427:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058428:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177048.058429:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058430:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177048.058431:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177048.058432:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177048.058434:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177048.058436:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177048.058438:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177048.058443:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856330368:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177048.058445:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058446:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177048.058448:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856330240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058453:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177048.058454:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177048.058455:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177048.058457:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177048.058458:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177048.058459:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177048.058460:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177048.058462:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:1.0:1786177048.058463:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177048.058465:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177048.058466:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:1.0:1786177048.058468:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177048.058468:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058469:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177048.058472:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856330368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058476:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177048.058477:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177048.058477:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177048.058479:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177048.058479:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177048.058480:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177048.058481:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177048.058482:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786177048.058483:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177048.058484:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177048.058485:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:1.0:1786177048.058486:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177048.058487:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058488:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058490:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177048.058491:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.058492:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177048.058493:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177048.058494:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177048.058496:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058500:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177048.058501:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177048.058502:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177048.058503:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177048.058504:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177048.058506:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856330496:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177048.058526:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177048.058527:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177048.058528:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177048.058530:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177048.058532:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177048.058533:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177048.058536:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177048.058538:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177048.058540:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177048.058542:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856330496, portal 4
00000100:00000001:1.0:1786177048.058544:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177048.058544:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:1.0:1786177048.058549:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058553:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177048.058555:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177048.058556:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177048.058558:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856330496, offset 0
00000400:00000200:1.0:1786177048.058560:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177048.058562:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177048.058565:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177048.058567:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177048.058569:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177048.058571:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177048.058572:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177048.058574:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254753:254753:256:4294967295] 192.168.202.110@tcp LPNI seq info [254742:254742:8:4294967295]
00000400:00000200:1.0:1786177048.058577:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177048.058581:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177048.058584:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177048.058586:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177048.058588:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177048.058590:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786177048.058593:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177048.058595:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786177048.058598:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177048.058599:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177048.058601:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177048.058603:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177048.058606:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177048.058607:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177048.058608:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177048.058610:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058615:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177048.058617:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058622:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.058623:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177048.058624:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177048.058627:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177048.058628:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177048.058630:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301943819
00000100:00000001:1.0:1786177048.058631:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301947597
00000100:00000001:1.0:1786177048.058633:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301943819
00000100:00000001:1.0:1786177048.058634:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301947597
00000100:00000001:1.0:1786177048.058636:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177048.058637:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.058638:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177048.058640:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177048/real 1786177048]  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177048.058645:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177048.058646:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177048.058647:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177048.058648:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177048.058649:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177048.058651:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177048.058653:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177048.058658:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058662:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177048.058663:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058664:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177048.058665:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177048.058667:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058671:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177048.058673:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058677:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177048.058678:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058679:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177048.058680:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058681:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177048.058682:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177048.058684:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177048.058686:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177048.058688:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177048.058690:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177048.058695:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856330496:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177048.058698:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177048.058700:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058704:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177048.058705:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177048.058707:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177048.058708:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177048.058709:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177048.058710:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856330624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177048.058712:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177048.058713:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177048.058714:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177048.058716:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177048.058717:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177048.058719:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177048.058721:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177048.058722:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177048.058724:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177048.058726:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856330624, portal 4
00000100:00000001:1.0:1786177048.058727:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177048.058728:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:1.0:1786177048.058732:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058736:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177048.058738:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177048.058739:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177048.058741:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856330624, offset 0
00000400:00000200:1.0:1786177048.058743:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177048.058745:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177048.058747:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177048.058748:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177048.058750:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177048.058752:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177048.058753:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177048.058755:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254754:254754:256:4294967295] 192.168.202.110@tcp LPNI seq info [254743:254743:8:4294967295]
00000400:00000200:1.0:1786177048.058758:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177048.058762:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177048.058764:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177048.058766:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177048.058768:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177048.058769:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786177048.058772:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177048.058774:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786177048.058778:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177048.058780:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177048.058782:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177048.058786:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177048.058788:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177048.058790:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177048.058793:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177048.058796:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058804:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177048.058807:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058814:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.058816:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177048.058819:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177048.058822:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177048.058826:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177048.058828:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301943827
00000100:00000001:1.0:1786177048.058830:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301947597
00000100:00000001:1.0:1786177048.058833:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301943827
00000100:00000001:1.0:1786177048.058835:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301947597
00000100:00000001:1.0:1786177048.058838:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177048.058839:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.058841:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177048.058845:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177048/real 1786177048]  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177048.058850:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177048.058851:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177048.058852:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177048.058854:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177048.058855:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177048.058857:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177048.058859:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177048.058864:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058868:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177048.058869:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058870:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177048.058871:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177048.058873:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058878:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177048.058880:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058884:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177048.058886:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058887:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177048.058887:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058889:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177048.058890:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177048.058892:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177048.058894:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177048.058896:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177048.058898:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177048.058903:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856330624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177048.058905:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058907:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177048.058908:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856330496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058912:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177048.058913:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177048.058915:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177048.058916:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177048.058917:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177048.058918:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177048.058919:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177048.058921:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:1.0:1786177048.058922:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177048.058923:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177048.058924:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:1.0:1786177048.058926:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177048.058926:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058927:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177048.058930:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856330624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177073 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177048.058933:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177048.058934:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177048.058935:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177048.058937:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177048.058938:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177048.058939:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177048.058940:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177048.058941:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:1.0:1786177048.058942:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177048.058943:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177048.058944:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:1.0:1786177048.058945:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177048.058946:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058947:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177048.058949:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177048.058950:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177048.058951:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177048.058952:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177048.632624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177048.632630:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177048.632634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177048.632635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177048.632638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177048.632639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177048.632692:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177048.632696:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177048.632699:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177049.080624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177049.080629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177049.080631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177049.080636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177049.080639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177049.080643:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177049.080645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177049.080647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177049.080649:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177049.656619:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177049.656624:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177049.656628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177049.656629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177049.656631:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177049.656632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177049.656685:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177049.656689:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177049.656692:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177050.104620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177050.104627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177050.104629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177050.104634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177050.104638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177050.104642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177050.104645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177050.104648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177050.104651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177050.680625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177050.680633:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177050.680635:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177050.680637:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177050.680641:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177050.680643:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177050.680645:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177050.680646:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177050.680649:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177050.680650:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177050.680650:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177050.680652:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177050.680653:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177050.680654:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177050.680655:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177050.680662:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177050.680664:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177050.680666:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177050.680721:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177050.680725:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177050.680728:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177051.128594:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177051.128599:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177051.128601:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177051.128606:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177051.128609:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177051.128612:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177051.128615:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177051.128617:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177051.128619:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177051.704598:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177051.704604:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177051.704608:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177051.704609:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177051.704612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177051.704613:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177051.704673:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177051.704678:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177051.704682:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177051.896607:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301947597, at 4301947648
00000100:00000040:1.0:1786177051.896607:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301947597, at 4301947648
00000100:00000010:0.0:1786177051.896625:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1d20.
00000100:00000010:1.0:1786177051.896627:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1720.
00000100:00000001:0.0:1786177052.152582:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177052.152588:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177052.152590:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177052.152594:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177052.152597:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177052.152600:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177052.152602:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177052.152604:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177052.152606:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177052.728647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177052.728655:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177052.728661:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177052.728662:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177052.728666:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177052.728669:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177052.728671:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177052.728675:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177052.728678:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177052.728681:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177052.728683:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177052.728686:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177052.728763:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177052.728768:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177052.728772:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786177053.176630:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177053.176642:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177053.176644:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177053.176648:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177053.176651:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177053.176657:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7276
00000400:00000200:3.0:1786177053.176665:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177053.176670:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177053.176674:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177053.176676:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177053.176678:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177053.176680:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177053.176682:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177053.176685:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177053.176686:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177053.176689:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177053.176691:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177053.176693:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177053.176695:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177053.176697:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177053.176698:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177053.176699:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177053.176701:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177053.176703:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177053.176705:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177053.176707:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177053.176709:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177053.176712:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177053.176714:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177053.176715:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177053.176716:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177053.176725:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d500.
00000020:00000040:3.0:1786177053.176727:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177053.176731:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177053.176732:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177053.176734:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177053.176736:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177053.176740:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177053.176759:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000001:3.0:1786177053.176767:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177053.176778:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177053.176790:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177053.176794:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177053.176800:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7c36393] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177053.176819:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177053.176822:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177053.176824:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177053.176826:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177053.176827:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177053.176829:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177053.176831:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7276
00000400:00000200:3.0:1786177053.176834:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177053.176836:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177053.176838:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177053.176841:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177053.176843:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177053.176845:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177053.176847:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177053.176849:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177053.176850:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177053.176851:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177053.176852:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:1.0:1786177053.176854:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1786177053.176854:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786177053.176858:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:1.0:1786177053.176859:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1786177053.176861:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177053.176863:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1786177053.176864:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177053.176868:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177053.176871:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:1.0:1786177053.176872:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177053.176873:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786177053.176880:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177053.176884:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177053.176888:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786177053.176892:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1786177053.176895:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177053.176896:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177053.176902:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.176904:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1786177053.176904:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:1.0:1786177053.176910:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:3.0:1786177053.176911:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177053.176915:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.176919:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177053.176919:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177053.176922:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f100.
00000020:00000040:3.0:1786177053.176927:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00100000:1.0:1786177053.176928:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856330752:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1786177053.176929:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177053.176932:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177053.176936:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177053.176938:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786177053.176941:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177053.176941:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177053.176947:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786177053.176948:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1786177053.176950:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.176952:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.176958:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:3.0:1786177053.176958:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:1.0:1786177053.176966:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
00000100:00000001:3.0:1786177053.176966:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177053.176970:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177053.176973:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1ef9f45] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177053.176978:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.176979:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177053.176980:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177053.176985:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177053.176986:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177053.176988:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1786177053.176989:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177053.176994:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:1.0:1786177053.176996:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00080000:3.0:1786177053.176998:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7276
00000400:00000200:3.0:1786177053.177002:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177053.177003:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000400:00000200:3.0:1786177053.177004:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177053.177008:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177053.177011:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:1.0:1786177053.177012:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856330752, portal 25
00000100:00000001:3.0:1786177053.177013:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177053.177017:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177053.177018:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177053.177019:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177053.177021:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000020:00000001:3.0:1786177053.177022:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177053.177026:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177053.177027:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177053.177029:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000040:1.0:1786177053.177032:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177053.177032:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177053.177036:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177053.177043:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177053.177046:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177053.177050:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177053.177051:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786177053.177052:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177053.177056:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000040:1.0:1786177053.177057:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786177053.177061:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177053.177063:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177053.177064:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177053.177067:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:1.0:1786177053.177070:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
02000000:00000001:3.0:1786177053.177071:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177053.177076:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177053.177078:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000200:1.0:1786177053.177082:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856330752, offset 0
00000008:00000001:3.0:1786177053.177082:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177053.177087:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177053.177093:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f480.
00000400:00000200:1.0:1786177053.177096:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:3.0:1786177053.177098:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177053.177103:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177053.177105:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177053.177108:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1786177053.177109:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786177053.177110:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177053.177115:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177053.177119:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000010:3.0:1786177053.177122:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000400:00000200:1.0:1786177053.177126:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1786177053.177128:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177053.177133:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00100000:3.0:1786177053.177139:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177053.177142:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177053.177149:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177053.177173:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254755:254755:256:4294967295] 192.168.202.110@tcp LPNI seq info [254744:254744:8:4294967295]
00000400:00000200:1.0:1786177053.177178:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177053.177185:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177053.177190:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177053.177194:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1786177053.177197:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092aa1aa7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1786177053.177208:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177053.177215:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786177053.177218:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000100:00080000:3.0:1786177053.177220:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177053.177226:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177053.177227:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177053.177229:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177053.177234:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.0:1786177053.177239:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786177053.177240:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7276
00000800:00000200:1.0:1786177053.177247:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000400:00000200:3.0:1786177053.177249:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:1.1:1786177053.177259:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:3.0:1786177053.177260:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177053.177265:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:1.1:1786177053.177266:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:3.0:1786177053.177271:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177053.177275:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:1.0:1786177053.177276:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786177053.177279:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000010:1.0:1786177053.177280:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000100:00000001:3.0:1786177053.177282:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177053.177286:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177053.177287:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177053.177288:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786177053.177290:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:3.0:1786177053.177291:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177053.177296:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177053.177299:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:1.0:1786177053.177302:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:3.0:1786177053.177307:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177053.177311:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786177053.177312:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000100:00000040:3.0:1786177053.177315:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:1.0:1786177053.177318:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:3.0:1786177053.177321:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177053.177326:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177053.177329:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177053.177332:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177053.177334:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1786177053.177335:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:3.0:1786177053.177338:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1786177053.177344:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177053.177344:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177053.177349:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177053.177353:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177053.177360:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.177362:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786177053.177362:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177053.177365:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ca80.
00000100:00000040:1.0:1786177053.177367:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177053.177369:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177053.177375:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177053.177377:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.177379:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177053.177380:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177053.177383:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786177053.177384:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:3.0:1786177053.177385:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177053.177390:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:3.0:1786177053.177393:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.177397:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177053.177398:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177053.177409:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177053.177410:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177053.177414:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.177416:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301952717
00000100:00000040:3.0:1786177053.177417:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ffb0865] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1786177053.177426:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1720.
00000100:00000001:3.0:1786177053.177427:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177053.177431:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177053.177434:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177053.177435:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177053.177436:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177053.177437:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177053.177438:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177053.177443:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7276
00000400:00000200:3.0:1786177053.177446:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177053.177449:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301952717
00000400:00000200:3.0:1786177053.177450:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000010:1.0:1786177053.177455:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1cc0.
00000100:00080000:3.0:1786177053.177455:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177053.177458:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177053.177461:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177053.177462:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177053.177466:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177053.177467:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177053.177469:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177053.177471:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177053.177472:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:1.0:1786177053.177474:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000020:00000040:3.0:1786177053.177475:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177053.177479:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177053.177481:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00100000:1.0:1786177053.177485:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177053/real 1786177053]  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177053.177485:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177053.177490:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177053.177497:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177053.177501:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177053.177504:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177053.177508:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177053.177531:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786177053.177534:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177053.177535:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177053.177537:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177053.177541:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177053.177543:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177053.177545:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177053.177546:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177053.177547:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177053.177551:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177053.177557:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69fb80.
00000400:00000001:1.0:1786177053.177559:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:3.0:1786177053.177560:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000400:00000001:1.0:1786177053.177564:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:3.0:1786177053.177565:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177053.177567:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177053.177570:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:3.0:1786177053.177570:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177053.177571:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177053.177574:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177053.177577:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
02000000:00000010:3.0:1786177053.177579:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:3.0:1786177053.177583:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177053.177589:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00100000:3.0:1786177053.177591:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177053.177595:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177053.177603:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177053.177605:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177053.177610:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177053.177620:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014f55bbb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177053.177629:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177053.177635:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7286)
00000100:00000200:1.0:1786177053.177640:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.177651:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177053.177653:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.177657:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177053.177659:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177053.177665:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.177672:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177053.177678:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.177689:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177053.177693:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.177696:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177053.177697:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.177700:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177053.177703:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177053.177706:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177053.177711:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177053.177714:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177053.177719:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177053.177725:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177053.177730:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00100000:1.0:1786177053.177731:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856330752:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177053.177732:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177053.177738:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786177053.177740:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177053.177742:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177053.177745:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.177746:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177053.177748:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177053.177751:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177053.177754:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856330752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177053.177756:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.177766:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177053.177768:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177053.177771:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177053.177776:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177053.177778:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177053.177780:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786177053.177782:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177053.177785:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786177053.177789:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177053.177791:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177053.177793:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:1.0:1786177053.177795:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177053.177797:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.177799:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.177806:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177053.177807:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.177810:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177053.177811:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177053.177813:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177053.177817:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.177825:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177053.177826:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.177828:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177053.177830:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177053.177832:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177053.177835:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856330880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177053.177839:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177053.177841:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177053.177843:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.177844:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177053.177848:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177053.177849:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177053.177853:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177053.177854:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177053.177857:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177053.177859:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856330880, portal 10
00000100:00000001:1.0:1786177053.177861:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177053.177862:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:1.0:1786177053.177867:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.177872:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177053.177873:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177053.177875:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177053.177876:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856330880, offset 0
00000400:00000200:1.0:1786177053.177879:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177053.177882:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177053.177884:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177053.177886:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177053.177888:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177053.177890:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177053.177892:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177053.177894:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254756:254756:256:4294967295] 192.168.202.110@tcp LPNI seq info [254745:254745:8:4294967295]
00000400:00000200:1.0:1786177053.177897:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177053.177901:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177053.177905:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177053.177909:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177053.177913:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177053.177916:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786177053.177921:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177053.177923:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786177053.177927:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177053.177929:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177053.177933:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177053.177936:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177053.177940:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177053.177942:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177053.177945:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177053.177949:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.177958:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177053.177961:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.177967:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.177968:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177053.177971:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177053.177974:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786177053.177976:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177053.177978:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301948938
00000100:00000001:1.0:1786177053.177980:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301952717
00000100:00000001:1.0:1786177053.177981:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301948938
00000100:00000001:1.0:1786177053.177983:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301952717
00000100:00000001:1.0:1786177053.177984:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177053.177986:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.177987:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177053.177990:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177053/real 1786177053]  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177053.177995:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177053.177996:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177053.177998:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177053.177999:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177053.178000:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177053.178002:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177053.178004:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177053.178009:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178014:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177053.178015:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178016:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177053.178017:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177053.178019:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178023:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177053.178025:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178029:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177053.178031:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178032:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177053.178033:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178034:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177053.178035:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177053.178037:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177053.178038:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177053.178040:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177053.178042:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177053.178047:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856330880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177053.178050:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177053.178052:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178057:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177053.178057:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.178059:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177053.178060:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177053.178061:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177053.178063:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177053.178065:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177053.178066:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177053.178067:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.178069:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177053.178071:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177053.178073:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177053.178075:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177053.178076:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177053.178078:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177053.178080:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856331008, portal 10
00000100:00000001:1.0:1786177053.178081:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177053.178082:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:1.0:1786177053.178086:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178090:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177053.178092:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177053.178093:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177053.178095:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856331008, offset 0
00000400:00000200:1.0:1786177053.178098:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177053.178100:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177053.178103:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177053.178105:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177053.178107:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177053.178108:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177053.178110:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177053.178112:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254757:254757:256:4294967295] 192.168.202.110@tcp LPNI seq info [254746:254746:8:4294967295]
00000400:00000200:1.0:1786177053.178115:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177053.178119:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177053.178122:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177053.178124:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177053.178126:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177053.178128:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786177053.178131:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177053.178133:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786177053.178136:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177053.178137:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177053.178139:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177053.178141:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177053.178143:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177053.178145:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177053.178146:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177053.178148:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178169:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177053.178173:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178177:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.178179:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177053.178182:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177053.178184:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177053.178186:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177053.178187:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301949063
00000100:00000001:1.0:1786177053.178189:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301952717
00000100:00000001:1.0:1786177053.178191:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301949063
00000100:00000001:1.0:1786177053.178193:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301952717
00000100:00000001:1.0:1786177053.178194:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177053.178196:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.178197:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177053.178199:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177053/real 1786177053]  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177053.178204:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177053.178205:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177053.178206:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177053.178207:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177053.178208:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177053.178210:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177053.178211:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177053.178217:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178221:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177053.178222:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178223:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177053.178224:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177053.178226:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178230:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177053.178232:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178237:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177053.178238:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178239:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177053.178240:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178241:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177053.178243:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177053.178244:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177053.178247:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177053.178248:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177053.178250:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177053.178255:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177053.178258:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177053.178260:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178264:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177053.178265:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.178267:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177053.178268:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177053.178269:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177053.178271:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177053.178273:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177053.178274:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177053.178275:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.178276:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177053.178278:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177053.178280:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177053.178282:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177053.178284:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177053.178288:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177053.178291:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856331136, portal 4
00000100:00000001:1.0:1786177053.178293:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177053.178294:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:1.0:1786177053.178300:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178307:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177053.178310:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177053.178312:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177053.178315:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856331136, offset 0
00000400:00000200:1.0:1786177053.178319:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177053.178323:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177053.178327:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177053.178330:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177053.178333:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177053.178335:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177053.178338:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177053.178341:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254758:254758:256:4294967295] 192.168.202.110@tcp LPNI seq info [254747:254747:8:4294967295]
00000400:00000200:1.0:1786177053.178344:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177053.178349:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177053.178352:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177053.178355:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177053.178358:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177053.178359:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786177053.178362:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177053.178364:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786177053.178367:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177053.178368:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177053.178370:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177053.178372:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177053.178375:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177053.178376:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177053.178377:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177053.178380:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178386:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177053.178387:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178399:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.178401:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177053.178404:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177053.178407:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177053.178410:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177053.178412:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301948938
00000100:00000001:1.0:1786177053.178414:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301952717
00000100:00000001:1.0:1786177053.178417:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301948938
00000100:00000001:1.0:1786177053.178419:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301952717
00000100:00000001:1.0:1786177053.178421:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177053.178423:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.178425:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177053.178430:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177053/real 1786177053]  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177053.178439:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177053.178441:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177053.178442:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177053.178446:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177053.178447:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177053.178449:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177053.178451:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177053.178457:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178461:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177053.178462:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178464:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177053.178464:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177053.178467:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178471:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177053.178474:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178479:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177053.178480:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178481:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177053.178482:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178483:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177053.178484:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177053.178485:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177053.178487:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177053.178489:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177053.178491:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177053.178496:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177053.178498:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177053.178500:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178504:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177053.178505:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.178507:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177053.178508:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177053.178526:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177053.178528:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177053.178530:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177053.178532:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177053.178534:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177053.178535:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177053.178538:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177053.178539:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177053.178542:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177053.178543:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177053.178545:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177053.178547:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856331264, portal 4
00000100:00000001:1.0:1786177053.178548:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177053.178549:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000040:1.0:1786177053.178554:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178559:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177053.178561:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177053.178564:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177053.178566:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856331264, offset 0
00000400:00000200:1.0:1786177053.178570:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177053.178574:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177053.178578:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177053.178581:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177053.178584:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177053.178586:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177053.178589:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177053.178592:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254759:254759:256:4294967295] 192.168.202.110@tcp LPNI seq info [254748:254748:8:4294967295]
00000400:00000200:1.0:1786177053.178596:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177053.178603:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177053.178608:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177053.178611:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177053.178616:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177053.178618:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786177053.178621:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177053.178624:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786177053.178629:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177053.178631:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177053.178634:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177053.178638:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177053.178641:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177053.178643:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177053.178645:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177053.178650:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178658:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177053.178662:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178669:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.178670:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177053.178674:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177053.178677:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177053.178680:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177053.178681:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301948947
00000100:00000001:1.0:1786177053.178683:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301952717
00000100:00000001:1.0:1786177053.178685:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301948947
00000100:00000001:1.0:1786177053.178686:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301952717
00000100:00000001:1.0:1786177053.178687:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177053.178689:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.178690:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177053.178693:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177053/real 1786177053]  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177053.178698:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177053.178699:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177053.178700:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177053.178702:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177053.178703:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177053.178705:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177053.178707:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177053.178712:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178716:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177053.178717:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178718:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177053.178719:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177053.178721:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178725:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177053.178727:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178731:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177053.178733:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178734:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177053.178735:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178736:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177053.178738:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177053.178739:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177053.178741:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177053.178743:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177053.178745:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177053.178749:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177053.178752:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178753:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177053.178756:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856330880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178760:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177053.178761:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177053.178763:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b01800.
02000000:00000001:1.0:1786177053.178765:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177053.178766:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177053.178766:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177053.178768:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177053.178769:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786177053.178771:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177053.178772:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177053.178773:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:1.0:1786177053.178775:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177053.178776:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178777:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177053.178779:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856331008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178783:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177053.178784:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177053.178785:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177053.178786:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177053.178787:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177053.178788:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177053.178789:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177053.178789:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:1.0:1786177053.178791:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177053.178791:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177053.178792:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:1.0:1786177053.178793:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177053.178794:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178795:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177053.178797:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856331136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178800:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177053.178802:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177053.178803:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03000.
02000000:00000001:1.0:1786177053.178804:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177053.178805:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177053.178806:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177053.178807:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177053.178808:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:1.0:1786177053.178809:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177053.178810:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177053.178810:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:1.0:1786177053.178812:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177053.178812:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178813:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177053.178815:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856331264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177053.178819:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177053.178819:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177053.178820:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177053.178822:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177053.178822:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177053.178823:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177053.178824:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177053.178825:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786177053.178826:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177053.178827:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177053.178827:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:1.0:1786177053.178829:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177053.178829:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178830:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177053.178833:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177053.178834:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177053.178835:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177053.178836:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177053.752615:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177053.752620:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177053.752624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177053.752625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177053.752628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177053.752629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177053.752680:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177053.752684:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177053.752687:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177054.200610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177054.200616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177054.200618:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177054.200622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177054.200625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177054.200629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177054.200631:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177054.200634:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177054.200636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177054.776639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177054.776647:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177054.776654:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177054.776656:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177054.776659:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177054.776661:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177054.776722:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177054.776726:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177054.776730:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177055.224616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177055.224622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177055.224624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177055.224628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177055.224631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177055.224634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177055.224636:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177055.224639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177055.224641:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177055.800642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177055.800649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177055.800656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177055.800657:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177055.800660:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177055.800662:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177055.800715:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177055.800718:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177055.800722:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177056.248638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177056.248645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177056.248646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177056.248651:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177056.248653:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177056.248657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177056.248659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177056.248661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177056.248664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177056.824630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177056.824636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177056.824641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177056.824643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177056.824647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177056.824648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177056.824712:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177056.824717:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177056.824720:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177057.016623:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301952717, at 4301952768
00000100:00000040:1.0:1786177057.016624:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301952717, at 4301952768
00000100:00000010:0.0:1786177057.016637:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1720.
00000100:00000010:1.0:1786177057.016641:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1cc0.
00000100:00000001:0.0:1786177057.272636:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177057.272641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177057.272643:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177057.272649:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177057.272651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177057.272655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177057.272657:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177057.272659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177057.272661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177057.848664:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177057.848672:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177057.848677:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177057.848677:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177057.848680:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177057.848681:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177057.848755:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177057.848762:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177057.848765:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786177058.296657:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177058.296672:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177058.296676:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177058.296681:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177058.296687:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177058.296696:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7281
00000400:00000200:0.0:1786177058.296707:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177058.296716:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177058.296721:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177058.296726:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.296730:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177058.296733:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.296736:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177058.296742:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177058.296745:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177058.296748:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177058.296752:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177058.296755:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177058.296759:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177058.296763:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.296765:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177058.296767:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.296770:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177058.296773:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177058.296777:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177058.296780:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177058.296782:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177058.296786:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177058.296789:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.296791:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177058.296793:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177058.296805:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419f80.
00000020:00000040:0.0:1786177058.296808:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177058.296813:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177058.296814:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.296817:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177058.296820:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177058.296825:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177058.296833:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000100:00000001:0.0:1786177058.296841:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177058.296856:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177058.296866:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177058.296870:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177058.296875:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ad753ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.296895:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177058.296897:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177058.296900:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177058.296902:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177058.296904:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177058.296906:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177058.296908:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7281
00000400:00000200:0.0:1786177058.296911:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177058.296913:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177058.296914:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177058.296916:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.296922:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177058.296924:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.296925:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177058.296928:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177058.296929:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177058.296930:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177058.296932:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177058.296934:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177058.296936:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177058.296939:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.296940:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177058.296942:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177058.296942:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177058.296947:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177058.296949:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1786177058.296950:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:1.0:1786177058.296953:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1786177058.296954:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177058.296958:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177058.296961:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1786177058.296962:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177058.296967:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177058.296972:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.296978:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177058.296979:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177058.296982:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1786177058.296983:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1786177058.296986:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:0.0:1786177058.296987:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177058.296990:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177058.296991:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177058.296997:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177058.296998:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418700.
00000020:00000040:0.0:1786177058.297002:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00100000:1.0:1786177058.297003:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856331392:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1786177058.297006:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1786177058.297009:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177058.297010:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.297013:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177058.297014:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177058.297015:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177058.297016:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177058.297019:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177058.297022:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177058.297024:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6000.
02000000:00000010:1.0:1786177058.297027:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b02c00.
00000100:00000001:0.0:1786177058.297028:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177058.297033:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177058.297038:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177058.297042:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177058.297049:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:0.0:1786177058.297050:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177058.297053:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177058.297057:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000040:0.0:1786177058.297058:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bf66f89d] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.297068:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177058.297068:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856331392, portal 25
00000100:00080000:0.0:1786177058.297070:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1786177058.297073:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1786177058.297075:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1786177058.297076:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761662848 : -114547947888768 : ffff97d1ba419f80)
00000100:00000001:0.0:1786177058.297079:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177058.297082:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:1.0:1786177058.297085:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.297086:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177058.297092:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7281
00000400:00000200:0.0:1786177058.297101:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177058.297102:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177058.297124:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177058.297126:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177058.297129:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856331392, offset 0
00000400:00000200:0.0:1786177058.297135:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1786177058.297140:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177058.297141:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177058.297146:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.297148:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177058.297150:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.297153:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177058.297155:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177058.297156:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177058.297159:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:1.0:1786177058.297159:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1786177058.297164:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177058.297167:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786177058.297169:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:0.0:1786177058.297173:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:1.0:1786177058.297177:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1786177058.297182:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177058.297185:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1786177058.297190:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1786177058.297192:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1786177058.297197:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:1.0:1786177058.297198:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.297204:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:1.0:1786177058.297207:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254760:254760:256:4294967295] 192.168.202.110@tcp LPNI seq info [254749:254749:8:4294967295]
00000020:00000040:0.0:1786177058.297210:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:1.0:1786177058.297216:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177058.297218:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177058.297223:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177058.297226:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177058.297231:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1786177058.297235:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:0.0:1786177058.297237:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.297243:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177058.297247:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000800:00000200:1.0:1786177058.297249:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000008:00000001:0.0:1786177058.297251:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.297256:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1786177058.297259:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000100:00000010:0.0:1786177058.297261:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418000.
00000020:00000040:0.0:1786177058.297266:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000800:00000200:1.0:1786177058.297270:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:0.0:1786177058.297272:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177058.297276:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.297279:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177058.297280:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177058.297283:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1786177058.297283:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
02000000:00000010:0.0:1786177058.297290:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000800:00000200:1.1:1786177058.297290:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177058.297297:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:0.0:1786177058.297299:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1786177058.297303:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177058.297306:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000100:00100000:0.0:1786177058.297311:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1786177058.297315:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177058.297325:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1786177058.297329:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1786177058.297332:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000100:00000001:0.0:1786177058.297335:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1786177058.297336:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177058.297344:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:0.0:1786177058.297346:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002e10e4f5] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177058.297351:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.297362:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177058.297367:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177058.297373:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177058.297375:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177058.297377:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177058.297377:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177058.297383:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:1.0:1786177058.297386:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177058.297387:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7281
00000400:00000200:0.0:1786177058.297396:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177058.297402:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177058.297407:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1786177058.297409:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177058.297411:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177058.297414:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177058.297415:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177058.297418:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.297420:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177058.297422:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177058.297423:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177058.297424:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:1.0:1786177058.297426:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177058.297427:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177058.297433:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177058.297435:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:1.0:1786177058.297437:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177058.297440:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.297443:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177058.297444:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301957837
00000100:00000040:0.0:1786177058.297446:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.297451:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000010:1.0:1786177058.297454:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1cc0.
00000020:00000040:0.0:1786177058.297455:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177058.297461:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177058.297464:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177058.297465:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:0.0:1786177058.297469:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177058.297475:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177058.297476:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301957837
02000000:00000001:0.0:1786177058.297479:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177058.297482:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1960.
00000100:00000001:0.0:1786177058.297484:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177058.297489:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1786177058.297490:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000008:00000001:0.0:1786177058.297494:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177058.297496:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177058.297500:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177058.297501:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177058.297505:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000010:0.0:1786177058.297507:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b800.
00000020:00000040:0.0:1786177058.297539:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00100000:1.0:1786177058.297542:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177058/real 1786177058]  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177058.297543:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177058.297546:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.297551:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177058.297554:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177058.297557:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177058.297564:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:0.0:1786177058.297569:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1786177058.297577:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00100000:0.0:1786177058.297578:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:1.0:1786177058.297582:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177058.297586:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000100:00000001:0.0:1786177058.297589:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177058.297593:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177058.297593:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177058.297599:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000040:0.0:1786177058.297601:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000192ee9a0] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177058.297601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1786177058.297605:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1786177058.297607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177058.297611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1786177058.297615:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.297617:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177058.297618:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:0.0:1786177058.297625:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786177058.297626:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177058.297630:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177058.297635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1786177058.297636:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177058.297639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.297640:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:1.0:1786177058.297640:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177058.297644:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177058.297645:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177058.297652:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177058.297657:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7281
00000100:00000001:1.0:1786177058.297658:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000400:00000200:0.0:1786177058.297662:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177058.297662:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177058.297667:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000400:00000200:0.0:1786177058.297669:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1786177058.297670:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177058.297673:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00100000:1.0:1786177058.297678:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177058.297682:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.297687:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177058.297691:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177058.297693:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.297695:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177058.297699:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000040:1.0:1786177058.297700:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177058.297701:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177058.297705:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177058.297708:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177058.297712:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786177058.297713:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:0.0:1786177058.297715:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:1.0:1786177058.297718:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177058.297721:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177058.297722:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177058.297726:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1786177058.297726:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177058.297730:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000040:0.0:1786177058.297731:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177058.297734:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:1.0:1786177058.297734:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:0.0:1786177058.297741:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:1.0:1786177058.297741:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177058.297748:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00080000:1.0:1786177058.297750:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177058.297751:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177058.297753:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:0.0:1786177058.297754:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177058.297759:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1786177058.297760:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177058.297763:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.297768:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177058.297771:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177058.297776:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177058.297777:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856331392:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177058.297778:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177058.297783:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418380.
00000020:00000040:0.0:1786177058.297788:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:1.0:1786177058.297790:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177058.297791:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177058.297793:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.297796:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177058.297797:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1786177058.297797:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1786177058.297800:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177058.297804:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000100:00000040:1.0:1786177058.297805:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419f80 x1872934856331392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.297808:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177058.297817:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177058.297826:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000001:0.0:1786177058.297828:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177058.297831:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177058.297833:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000040:0.0:1786177058.297834:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e6b901d7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1786177058.297837:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b02c00.
02000000:00000001:1.0:1786177058.297843:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:0.0:1786177058.297845:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1786177058.297847:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177058.297849:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000040:0.0:1786177058.297850:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7291)
00000020:00000001:1.0:1786177058.297852:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177058.297855:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:1.0:1786177058.297860:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177058.297861:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177058.297863:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419f80.
00000100:00000001:1.0:1786177058.297866:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177058.297867:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177058.297869:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177058.297877:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177058.297878:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177058.297883:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177058.297884:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177058.297907:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177058.297912:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.297919:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177058.297921:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177058.297923:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177058.297924:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177058.297926:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177058.297928:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331520:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177058.297931:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177058.297933:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177058.297934:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177058.297936:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177058.297939:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177058.297941:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177058.297944:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177058.297947:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c3c8.
00000400:00000010:0.0:1786177058.297953:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c44c8.
00000100:00000200:0.0:1786177058.297955:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856331520, portal 10
00000100:00000001:0.0:1786177058.297957:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177058.297958:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761656576 : -114547947895040 : ffff97d1ba418700)
00000100:00000040:0.0:1786177058.297964:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.297970:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177058.297972:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177058.297973:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786177058.297975:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856331520, offset 0
00000400:00000200:0.0:1786177058.297980:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177058.297984:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177058.297988:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177058.297991:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177058.297994:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177058.297997:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177058.298000:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177058.298003:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254761:254761:256:4294967295] 192.168.202.110@tcp LPNI seq info [254750:254750:8:4294967295]
00000400:00000200:0.0:1786177058.298008:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177058.298015:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177058.298020:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177058.298025:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177058.298030:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177058.298033:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786177058.298038:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177058.298042:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786177058.298047:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177058.298049:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177058.298053:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177058.298057:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177058.298061:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786177058.298063:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786177058.298067:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177058.298071:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298080:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177058.298084:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298100:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.298102:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177058.298119:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177058.298124:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177058.298127:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177058.298129:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301954058
00000100:00000001:0.0:1786177058.298131:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301957837
00000100:00000001:0.0:1786177058.298133:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301954058
00000100:00000001:0.0:1786177058.298135:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301957837
00000100:00000001:0.0:1786177058.298136:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177058.298138:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.298139:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177058.298142:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177058/real 1786177058]  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177058.298148:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177058.298149:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177058.298151:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c3c8.
00000400:00000200:0.0:1786177058.298153:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c44c8
00000400:00000010:0.0:1786177058.298154:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c44c8.
00000100:00000001:0.0:1786177058.298156:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177058.298158:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177058.298164:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298168:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177058.298169:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298170:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177058.298171:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177058.298173:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298179:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177058.298181:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298186:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177058.298188:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298189:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177058.298189:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298191:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177058.298192:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177058.298195:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177058.298198:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177058.298200:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177058.298204:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177058.298212:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331520:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177058.298216:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177058.298219:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298227:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177058.298229:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177058.298231:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177058.298233:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177058.298235:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177058.298239:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177058.298243:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177058.298245:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177058.298247:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177058.298249:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177058.298253:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177058.298255:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177058.298258:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177058.298260:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c3c8.
00000400:00000010:0.0:1786177058.298262:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c44c8.
00000100:00000200:0.0:1786177058.298264:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856331648, portal 10
00000100:00000001:0.0:1786177058.298266:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177058.298267:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761654784 : -114547947896832 : ffff97d1ba418000)
00000100:00000040:0.0:1786177058.298272:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298278:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177058.298280:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177058.298283:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786177058.298285:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856331648, offset 0
00000400:00000200:0.0:1786177058.298289:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177058.298294:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177058.298298:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177058.298301:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177058.298304:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177058.298307:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177058.298310:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177058.298314:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254762:254762:256:4294967295] 192.168.202.110@tcp LPNI seq info [254751:254751:8:4294967295]
00000400:00000200:0.0:1786177058.298319:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177058.298336:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177058.298340:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177058.298346:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177058.298349:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177058.298351:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786177058.298354:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177058.298356:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786177058.298359:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177058.298361:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177058.298363:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177058.298366:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177058.298368:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786177058.298370:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786177058.298372:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177058.298374:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298381:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177058.298383:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298387:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.298388:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177058.298391:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177058.298393:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177058.298396:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177058.298397:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301954183
00000100:00000001:0.0:1786177058.298399:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301957837
00000100:00000001:0.0:1786177058.298400:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301954183
00000100:00000001:0.0:1786177058.298402:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301957837
00000100:00000001:0.0:1786177058.298403:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177058.298405:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.298406:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177058.298408:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177058/real 1786177058]  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177058.298413:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177058.298414:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177058.298415:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c3c8.
00000400:00000200:0.0:1786177058.298417:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c44c8
00000400:00000010:0.0:1786177058.298418:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c44c8.
00000100:00000001:0.0:1786177058.298420:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177058.298422:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177058.298428:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298432:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177058.298433:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298434:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177058.298435:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177058.298437:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298441:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177058.298443:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298450:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177058.298452:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298454:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177058.298455:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298457:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177058.298459:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177058.298461:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177058.298464:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177058.298467:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177058.298471:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177058.298480:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177058.298484:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177058.298489:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298496:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177058.298498:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177058.298500:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177058.298502:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177058.298504:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177058.298506:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177058.298509:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177058.298528:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177058.298530:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177058.298531:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177058.298534:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4800.
02000000:00000001:0.0:1786177058.298536:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177058.298538:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177058.298540:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c3c8.
00000400:00000010:0.0:1786177058.298543:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c44c8.
00000100:00000200:0.0:1786177058.298546:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856331776, portal 4
00000100:00000001:0.0:1786177058.298547:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177058.298548:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761669120 : -114547947882496 : ffff97d1ba41b800)
00000100:00000040:0.0:1786177058.298554:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298558:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177058.298560:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177058.298562:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786177058.298563:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856331776, offset 0
00000400:00000200:0.0:1786177058.298566:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177058.298569:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177058.298571:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177058.298573:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177058.298575:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177058.298577:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177058.298578:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177058.298581:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254763:254763:256:4294967295] 192.168.202.110@tcp LPNI seq info [254752:254752:8:4294967295]
00000400:00000200:0.0:1786177058.298584:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177058.298589:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177058.298592:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177058.298594:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177058.298597:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177058.298599:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786177058.298602:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177058.298604:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786177058.298606:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177058.298608:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177058.298610:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177058.298613:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177058.298615:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786177058.298616:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786177058.298618:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177058.298620:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298626:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177058.298628:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298632:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.298633:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177058.298635:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177058.298637:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177058.298640:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177058.298641:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301954059
00000100:00000001:0.0:1786177058.298644:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301957837
00000100:00000001:0.0:1786177058.298645:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301954059
00000100:00000001:0.0:1786177058.298647:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301957837
00000100:00000001:0.0:1786177058.298648:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177058.298650:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.298651:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177058.298654:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177058/real 1786177058]  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177058.298659:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177058.298660:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177058.298661:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c3c8.
00000400:00000200:0.0:1786177058.298662:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c44c8
00000400:00000010:0.0:1786177058.298663:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c44c8.
00000100:00000001:0.0:1786177058.298665:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177058.298667:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177058.298672:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298676:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177058.298677:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298678:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177058.298679:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177058.298681:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298685:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177058.298687:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298691:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177058.298693:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298694:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177058.298695:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298697:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177058.298698:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177058.298700:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177058.298702:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177058.298704:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177058.298707:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177058.298712:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177058.298714:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177058.298716:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298719:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177058.298720:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177058.298722:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177058.298723:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177058.298724:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177058.298726:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331904:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177058.298727:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177058.298728:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177058.298729:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177058.298730:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177058.298732:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177058.298733:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177058.298735:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177058.298737:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c3c8.
00000400:00000010:0.0:1786177058.298739:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c44c8.
00000100:00000200:0.0:1786177058.298741:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856331904, portal 4
00000100:00000001:0.0:1786177058.298742:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177058.298743:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761655680 : -114547947895936 : ffff97d1ba418380)
00000100:00000040:0.0:1786177058.298747:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298752:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177058.298753:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177058.298755:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786177058.298756:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856331904, offset 0
00000400:00000200:0.0:1786177058.298759:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177058.298761:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177058.298763:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177058.298765:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177058.298767:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177058.298769:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177058.298770:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177058.298772:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254764:254764:256:4294967295] 192.168.202.110@tcp LPNI seq info [254753:254753:8:4294967295]
00000400:00000200:0.0:1786177058.298775:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177058.298778:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177058.298782:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177058.298784:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177058.298786:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177058.298788:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786177058.298791:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177058.298792:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786177058.298795:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177058.298796:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177058.298798:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177058.298801:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177058.298803:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786177058.298804:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786177058.298805:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177058.298808:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298813:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177058.298815:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298819:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.298821:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177058.298823:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177058.298825:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177058.298827:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177058.298829:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301954067
00000100:00000001:0.0:1786177058.298831:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301957837
00000100:00000001:0.0:1786177058.298832:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301954067
00000100:00000001:0.0:1786177058.298834:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301957837
00000100:00000001:0.0:1786177058.298835:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177058.298836:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.298837:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177058.298839:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177058/real 1786177058]  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177058.298843:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177058.298844:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177058.298845:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c3c8.
00000400:00000200:0.0:1786177058.298846:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c44c8
00000400:00000010:0.0:1786177058.298847:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c44c8.
00000100:00000001:0.0:1786177058.298849:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177058.298851:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177058.298856:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298860:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177058.298861:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298862:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177058.298863:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177058.298865:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298869:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177058.298871:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298876:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177058.298878:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298879:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177058.298880:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298881:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177058.298883:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177058.298884:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177058.298886:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177058.298887:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177058.298889:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177058.298894:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856331904:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177058.298896:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298897:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177058.298899:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418700 x1872934856331520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298903:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177058.298904:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177058.298905:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177058.298907:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177058.298908:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177058.298909:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177058.298910:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177058.298911:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786177058.298913:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177058.298914:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177058.298915:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418700.
00000100:00000001:0.0:1786177058.298917:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177058.298918:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298920:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177058.298922:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418000 x1872934856331648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298926:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177058.298927:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177058.298928:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177058.298929:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177058.298930:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177058.298931:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177058.298932:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177058.298933:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786177058.298934:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177058.298934:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177058.298935:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418000.
00000100:00000001:0.0:1786177058.298936:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177058.298937:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298938:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177058.298940:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b800 x1872934856331776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298943:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177058.298944:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177058.298945:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4800.
02000000:00000001:0.0:1786177058.298946:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177058.298947:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177058.298947:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177058.298948:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177058.298949:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177058.298950:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177058.298951:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177058.298952:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b800.
00000100:00000001:0.0:1786177058.298953:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177058.298954:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298955:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177058.298957:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418380 x1872934856331904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177058.298965:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177058.298965:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177058.298966:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177058.298968:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177058.298968:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177058.298969:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177058.298970:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177058.298971:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786177058.298972:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177058.298973:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177058.298973:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418380.
00000100:00000001:0.0:1786177058.298975:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177058.298975:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298976:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177058.298979:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177058.298980:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177058.298981:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177058.298983:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177058.872665:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177058.872673:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177058.872678:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177058.872679:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177058.872682:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177058.872683:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177058.872749:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177058.872754:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177058.872757:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177059.320679:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177059.320689:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177059.320691:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177059.320696:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177059.320700:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177059.320704:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177059.320707:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177059.320710:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177059.320712:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177059.896643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177059.896650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177059.896656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177059.896658:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177059.896662:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177059.896664:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177059.896744:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177059.896753:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177059.896759:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177060.344671:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177060.344683:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177060.344687:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177060.344694:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177060.344699:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177060.344705:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177060.344709:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177060.344713:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177060.344716:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177060.920631:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177060.920637:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177060.920639:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177060.920642:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177060.920646:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177060.920648:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177060.920650:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177060.920651:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177060.920654:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177060.920655:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177060.920656:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177060.920657:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177060.920658:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177060.920659:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177060.920660:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177060.920662:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177060.920664:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177060.920665:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177060.920715:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177060.920718:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177060.920721:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177061.368613:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177061.368620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177061.368621:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177061.368626:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177061.368628:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177061.368631:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177061.368634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177061.368636:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177061.368638:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177061.944611:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177061.944617:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177061.944621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177061.944622:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177061.944624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177061.944625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177061.944668:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177061.944671:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177061.944674:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177062.136619:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301957837, at 4301957888
00000100:00000040:1.0:1786177062.136619:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301957837, at 4301957888
00000100:00000010:1.0:1786177062.136636:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1960.
00000100:00000010:0.0:1786177062.136638:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1cc0.
00000100:00000001:2.0:1786177062.392631:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177062.392641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177062.392644:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177062.392649:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177062.392651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177062.392654:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177062.392656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177062.392658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177062.392661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177062.968627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177062.968632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177062.968636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177062.968637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177062.968639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177062.968642:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177062.968644:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177062.968646:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177062.968649:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177062.968650:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177062.968652:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177062.968654:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177062.968728:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177062.968731:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177062.968734:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177063.416661:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177063.416670:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177063.416673:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177063.416673:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1786177063.416682:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:3.0:1786177063.416683:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177063.416687:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1786177063.416689:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1786177063.416693:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177063.416695:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177063.416700:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177063.416700:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786177063.416705:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177063.416709:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177063.416711:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7286
00000400:00000200:3.0:1786177063.416723:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177063.416731:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177063.416736:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177063.416740:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177063.416744:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177063.416746:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177063.416749:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177063.416752:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177063.416755:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177063.416757:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177063.416761:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177063.416763:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177063.416766:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177063.416770:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177063.416772:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177063.416774:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177063.416776:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177063.416780:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177063.416784:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177063.416787:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177063.416790:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177063.416794:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177063.416797:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177063.416799:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177063.416801:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177063.416811:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f800.
00000020:00000040:3.0:1786177063.416815:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177063.416820:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177063.416821:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177063.416825:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177063.416828:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177063.416832:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177063.416840:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786177063.416848:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177063.416859:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177063.416877:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177063.416882:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177063.416888:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4d339d9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177063.416915:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177063.416918:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177063.416922:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177063.416923:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177063.416925:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177063.416928:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177063.416932:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7286
00000400:00000200:3.0:1786177063.416937:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177063.416941:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177063.416943:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177063.416947:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177063.416950:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177063.416952:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1786177063.416955:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177063.416956:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177063.416960:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1786177063.416961:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177063.416967:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000040:0.0:1786177063.416969:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177063.416970:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177063.416975:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177063.416979:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:0.0:1786177063.416982:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177063.416986:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177063.416986:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:0.0:1786177063.416991:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:3.0:1786177063.416993:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:0.0:1786177063.416994:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177063.417000:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177063.417004:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856332032:192.168.202.110@tcp:250:kworker.0
00000100:00000040:3.0:1786177063.417004:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177063.417012:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1786177063.417012:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1786177063.417017:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:3.0:1786177063.417020:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:0.0:1786177063.417021:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177063.417026:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1786177063.417027:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000010:0.0:1786177063.417031:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
00000020:00000040:3.0:1786177063.417033:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
02000000:00000001:0.0:1786177063.417034:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177063.417041:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1786177063.417044:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177063.417049:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c3c8.
00000400:00000010:0.0:1786177063.417066:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c44c8.
00000100:00000001:3.0:1786177063.417072:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177063.417076:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:0.0:1786177063.417077:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856332032, portal 25
00000100:00000001:0.0:1786177063.417080:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177063.417082:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
02000000:00000001:3.0:1786177063.417083:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1786177063.417090:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177063.417090:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177063.417096:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177063.417102:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1786177063.417106:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000008:00000001:3.0:1786177063.417108:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177063.417113:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177063.417114:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177063.417118:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000010:3.0:1786177063.417122:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d880.
00000100:00000200:0.0:1786177063.417124:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856332032, offset 0
00000020:00000040:3.0:1786177063.417127:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177063.417132:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177063.417136:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177063.417137:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177063.417140:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177063.417143:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177063.417145:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177063.417150:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:3.0:1786177063.417152:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000400:00000200:0.0:1786177063.417158:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1786177063.417159:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177063.417165:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:3.0:1786177063.417170:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1786177063.417171:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177063.417176:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177063.417181:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177063.417189:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254765:254765:256:4294967295] 192.168.202.110@tcp LPNI seq info [254754:254754:8:4294967295]
00000100:00000001:3.0:1786177063.417192:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177063.417198:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177063.417199:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1786177063.417207:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097f9b8db] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177063.417218:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786177063.417230:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177063.417234:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786177063.417236:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:0.0:1786177063.417242:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000100:00080000:3.0:1786177063.417243:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177063.417248:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177063.417251:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:0.0:1786177063.417252:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177063.417257:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:3.0:1786177063.417259:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177063.417266:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7286
00000800:00000200:0.1:1786177063.417268:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177063.417273:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000400:00000200:3.0:1786177063.417278:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000100:0.0:1786177063.417280:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177063.417284:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:3.0:1786177063.417288:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:0.0:1786177063.417290:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:3.0:1786177063.417293:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786177063.417297:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:3.0:1786177063.417302:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177063.417304:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000100:00000001:3.0:1786177063.417308:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1786177063.417309:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786177063.417315:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:3.0:1786177063.417315:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177063.417321:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:0.0:1786177063.417322:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177063.417327:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177063.417330:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000100:00000001:0.0:1786177063.417335:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000001:3.0:1786177063.417335:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177063.417339:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000040:0.0:1786177063.417341:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177063.417345:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177063.417352:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:0.0:1786177063.417353:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.417357:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:3.0:1786177063.417362:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177063.417363:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177063.417368:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786177063.417371:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177063.417376:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301962957
00000100:00000040:3.0:1786177063.417376:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177063.417382:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000010:0.0:1786177063.417384:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000020:00000040:3.0:1786177063.417388:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000001:0.0:1786177063.417392:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:3.0:1786177063.417394:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177063.417398:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301962957
00000100:00000001:3.0:1786177063.417399:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177063.417402:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134780.
02000000:00000001:3.0:1786177063.417404:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177063.417406:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:3.0:1786177063.417410:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177063.417411:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177063.417417:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177063.417418:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.417420:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1786177063.417421:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00100000:0.0:1786177063.417426:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177063/real 1786177063]  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1786177063.417426:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177063.417434:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177063.417439:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177063.417447:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c380.
00000400:00000001:0.0:1786177063.417449:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177063.417451:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:3.0:1786177063.417452:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000010:0.0:1786177063.417454:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c3c8.
02000000:00000001:3.0:1786177063.417459:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1786177063.417460:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c44c8
00000400:00000010:0.0:1786177063.417464:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c44c8.
02000000:00000001:3.0:1786177063.417464:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.417469:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177063.417472:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177063.417477:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177063.417481:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786177063.417483:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177063.417485:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177063.417485:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1786177063.417490:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1786177063.417490:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.417493:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177063.417497:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1786177063.417499:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:3.0:1786177063.417504:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.417529:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177063.417534:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.417539:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177063.417542:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.417544:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00100000:3.0:1786177063.417545:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177063.417546:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.417550:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177063.417553:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177063.417558:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177063.417566:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177063.417566:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177063.417570:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1786177063.417571:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177063.417576:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177063.417582:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000833bf1c9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177063.417588:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856332032:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177063.417596:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.417600:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786177063.417603:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177063.417605:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856332032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786177063.417609:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177063.417613:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00080000:3.0:1786177063.417616:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
02000000:00000001:0.0:1786177063.417617:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:3.0:1786177063.417621:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000010:0.0:1786177063.417622:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786177063.417627:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00080000:3.0:1786177063.417627:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000020:00000001:0.0:1786177063.417630:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177063.417633:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:3.0:1786177063.417634:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000020:00000001:0.0:1786177063.417635:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177063.417639:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
00000100:00080000:3.0:1786177063.417643:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7286
02000000:00000001:0.0:1786177063.417644:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177063.417646:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177063.417649:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000400:00000200:3.0:1786177063.417653:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177063.417654:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177063.417656:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.417659:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1786177063.417661:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177063.417666:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177063.417667:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177063.417670:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177063.417671:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177063.417673:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177063.417676:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177063.417676:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177063.417680:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177063.417682:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:0.0:1786177063.417685:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177063.417687:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177063.417693:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177063.417695:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:0.0:1786177063.417697:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177063.417700:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177063.417700:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
02000000:00000001:0.0:1786177063.417704:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:3.0:1786177063.417705:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
02000000:00000001:0.0:1786177063.417706:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177063.417713:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177063.417713:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00100000:0.0:1786177063.417719:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856332160:192.168.202.110@tcp:38:kworker.0
00000020:00000001:3.0:1786177063.417720:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:0.0:1786177063.417727:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:3.0:1786177063.417728:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:0.0:1786177063.417732:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1786177063.417732:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:0.0:1786177063.417736:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177063.417737:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:0.0:1786177063.417741:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1786177063.417742:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000010:0.0:1786177063.417747:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
00000020:00000040:3.0:1786177063.417748:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
02000000:00000001:0.0:1786177063.417755:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177063.417755:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177063.417761:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177063.417763:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786177063.417765:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786177063.417767:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:0.0:1786177063.417771:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c3c8.
02000000:00000001:3.0:1786177063.417772:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177063.417776:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000010:0.0:1786177063.417780:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c44c8.
00000008:00100000:3.0:1786177063.417781:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000200:0.0:1786177063.417786:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856332160, portal 10
00000008:00000001:3.0:1786177063.417786:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177063.417788:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.417790:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177063.417793:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
00000100:00000010:3.0:1786177063.417795:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ed80.
00000100:00000040:0.0:1786177063.417801:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177063.417802:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177063.417809:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1786177063.417811:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:3.0:1786177063.417813:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177063.417816:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177063.417819:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:0.0:1786177063.417820:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
02000000:00000001:3.0:1786177063.417823:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:0.0:1786177063.417824:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856332160, offset 0
02000000:00000001:3.0:1786177063.417828:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177063.417831:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177063.417836:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:3.0:1786177063.417836:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000400:00000200:0.0:1786177063.417840:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1786177063.417843:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177063.417846:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177063.417853:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00100000:3.0:1786177063.417854:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1786177063.417857:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177063.417864:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177063.417872:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254766:254766:256:4294967295] 192.168.202.110@tcp LPNI seq info [254755:254755:8:4294967295]
00000100:00000001:3.0:1786177063.417873:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177063.417877:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177063.417883:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1786177063.417886:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ccd6c6d2] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177063.417895:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177063.417908:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177063.417914:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000100:00000001:3.0:1786177063.417915:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177063.417921:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786177063.417921:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.0:1786177063.417927:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00080000:3.0:1786177063.417929:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:0.1:1786177063.417933:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177063.417934:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.1:1786177063.417937:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00080000:3.0:1786177063.417938:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000100:0.0:1786177063.417943:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177063.417946:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000100:00000001:3.0:1786177063.417946:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1786177063.417952:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:3.0:1786177063.417955:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7286
00000400:00000200:0.0:1786177063.417958:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177063.417964:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000200:3.0:1786177063.417967:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177063.417968:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786177063.417972:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:3.0:1786177063.417977:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000200:0.0:1786177063.417978:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786177063.417983:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177063.417992:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177063.417992:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177063.417997:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177063.418000:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177063.418003:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177063.418011:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177063.418011:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177063.418016:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:3.0:1786177063.418018:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:0.0:1786177063.418019:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177063.418021:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000040:0.0:1786177063.418024:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000020:00000001:3.0:1786177063.418025:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177063.418030:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000400:00000200:0.0:1786177063.418031:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177063.418036:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301959178
00000020:00000001:3.0:1786177063.418036:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:0.0:1786177063.418040:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301962957
00000100:00000001:0.0:1786177063.418044:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301959178
00000020:00000001:3.0:1786177063.418044:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:0.0:1786177063.418048:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301962957
00000100:00000001:0.0:1786177063.418051:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177063.418068:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177063.418068:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177063.418072:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1786177063.418075:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:0.0:1786177063.418080:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177063/real 1786177063]  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177063.418083:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177063.418090:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000001:0.0:1786177063.418095:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:3.0:1786177063.418096:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000400:00000001:0.0:1786177063.418098:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:3.0:1786177063.418103:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:0.0:1786177063.418104:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c3c8.
00000400:00000200:0.0:1786177063.418110:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c44c8
00000100:00000001:3.0:1786177063.418110:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177063.418116:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c44c8.
02000000:00000001:3.0:1786177063.418116:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177063.418121:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:3.0:1786177063.418122:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:0.0:1786177063.418128:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177063.418128:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177063.418133:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177063.418137:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177063.418144:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177063.418148:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786177063.418150:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1786177063.418154:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e680.
00000100:00000001:0.0:1786177063.418160:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000040:3.0:1786177063.418161:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:0.0:1786177063.418163:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177063.418166:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1786177063.418167:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1786177063.418171:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.418172:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177063.418178:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177063.418178:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177063.418181:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177063.418184:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.418191:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:3.0:1786177063.418192:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000040:0.0:1786177063.418199:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177063.418199:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177063.418209:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177063.418213:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177063.418219:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418223:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177063.418227:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177063.418228:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177063.418231:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177063.418233:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.418234:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177063.418239:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00000040:3.0:1786177063.418241:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1839131] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177063.418244:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177063.418248:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177063.418256:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177063.418264:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177063.418271:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856332160:192.168.202.110@tcp:38:kworker.0
00000100:00000040:3.0:1786177063.418274:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7296)
00000100:00000001:0.0:1786177063.418275:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177063.418280:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418286:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177063.418287:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177063.418288:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177063.418290:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177063.418292:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177063.418295:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856332288:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177063.418297:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177063.418299:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177063.418301:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177063.418302:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177063.418305:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177063.418306:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177063.418309:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177063.418310:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c3c8.
00000400:00000010:0.0:1786177063.418313:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c44c8.
00000100:00000200:0.0:1786177063.418315:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856332288, portal 10
00000100:00000001:0.0:1786177063.418316:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177063.418317:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764293504 : -114547945258112 : ffff97d1ba69c380)
00000100:00000040:0.0:1786177063.418321:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418326:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177063.418327:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177063.418329:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786177063.418330:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856332288, offset 0
00000400:00000200:0.0:1786177063.418333:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177063.418335:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177063.418338:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177063.418340:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177063.418342:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177063.418344:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177063.418346:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177063.418348:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254767:254767:256:4294967295] 192.168.202.110@tcp LPNI seq info [254756:254756:8:4294967295]
00000400:00000200:0.0:1786177063.418350:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177063.418355:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177063.418357:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177063.418359:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177063.418362:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177063.418364:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786177063.418367:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177063.418368:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1786177063.418371:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177063.418372:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177063.418374:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177063.418377:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177063.418379:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786177063.418380:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786177063.418382:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177063.418385:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418390:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177063.418392:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418396:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.418397:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177063.418399:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177063.418402:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177063.418404:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177063.418405:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301959303
00000100:00000001:0.0:1786177063.418407:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301962957
00000100:00000001:0.0:1786177063.418409:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301959303
00000100:00000001:0.0:1786177063.418410:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301962957
00000100:00000001:0.0:1786177063.418412:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177063.418413:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.418415:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177063.418417:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177063/real 1786177063]  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177063.418422:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177063.418423:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177063.418424:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c3c8.
00000400:00000200:0.0:1786177063.418425:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c44c8
00000400:00000010:0.0:1786177063.418426:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c44c8.
00000100:00000001:0.0:1786177063.418427:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177063.418429:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177063.418434:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418439:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177063.418439:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418441:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177063.418441:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177063.418443:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418447:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177063.418450:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418454:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177063.418455:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418456:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177063.418457:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418459:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177063.418460:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177063.418462:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177063.418464:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177063.418465:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177063.418468:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177063.418472:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856332288:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177063.418475:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418476:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177063.418478:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856332160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418481:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177063.418483:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177063.418484:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786177063.418485:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177063.418487:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177063.418488:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177063.418489:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177063.418490:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:0.0:1786177063.418492:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177063.418493:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177063.418494:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:0.0:1786177063.418496:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177063.418497:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418498:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177063.418500:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c380 x1872934856332288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418504:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177063.418505:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177063.418506:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177063.418507:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177063.418508:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177063.418524:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177063.418526:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177063.418527:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786177063.418529:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177063.418530:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177063.418531:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c380.
00000100:00000001:0.0:1786177063.418532:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177063.418533:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418534:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418537:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177063.418538:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.418540:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177063.418541:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177063.418542:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177063.418544:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418549:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177063.418550:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177063.418551:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177063.418552:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177063.418554:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177063.418556:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856332416:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177063.418557:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177063.418559:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177063.418560:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177063.418561:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177063.418563:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177063.418564:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177063.418567:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177063.418569:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c3c8.
00000400:00000010:0.0:1786177063.418571:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c44c8.
00000100:00000200:0.0:1786177063.418573:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856332416, portal 4
00000100:00000001:0.0:1786177063.418574:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177063.418576:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:0.0:1786177063.418580:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418584:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177063.418586:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177063.418587:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786177063.418589:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856332416, offset 0
00000400:00000200:0.0:1786177063.418591:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177063.418593:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177063.418595:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177063.418597:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177063.418599:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177063.418601:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177063.418602:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177063.418605:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254768:254768:256:4294967295] 192.168.202.110@tcp LPNI seq info [254757:254757:8:4294967295]
00000400:00000200:0.0:1786177063.418607:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177063.418612:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177063.418615:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177063.418617:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177063.418619:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177063.418621:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786177063.418623:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177063.418625:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1786177063.418628:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177063.418629:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177063.418631:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177063.418633:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177063.418635:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786177063.418636:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786177063.418638:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177063.418640:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418645:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177063.418647:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418652:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.418653:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177063.418655:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177063.418657:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177063.418659:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177063.418660:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301959179
00000100:00000001:0.0:1786177063.418662:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301962957
00000100:00000001:0.0:1786177063.418663:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301959179
00000100:00000001:0.0:1786177063.418665:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301962957
00000100:00000001:0.0:1786177063.418666:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177063.418667:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.418668:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177063.418670:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177063/real 1786177063]  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177063.418675:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177063.418676:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177063.418678:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c3c8.
00000400:00000200:0.0:1786177063.418679:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c44c8
00000400:00000010:0.0:1786177063.418680:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c44c8.
00000100:00000001:0.0:1786177063.418681:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177063.418683:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177063.418689:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418693:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177063.418694:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418695:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177063.418696:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177063.418698:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418703:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177063.418705:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418709:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177063.418711:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418712:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177063.418712:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418714:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177063.418714:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177063.418716:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177063.418718:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177063.418719:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177063.418721:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177063.418729:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856332416:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177063.418732:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177063.418735:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418741:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177063.418742:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177063.418745:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177063.418746:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177063.418748:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177063.418751:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856332544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177063.418755:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177063.418757:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177063.418759:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177063.418761:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177063.418764:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786177063.418766:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177063.418770:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177063.418773:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c3c8.
00000400:00000010:0.0:1786177063.418777:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c44c8.
00000100:00000200:0.0:1786177063.418780:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856332544, portal 4
00000100:00000001:0.0:1786177063.418781:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177063.418783:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00000040:0.0:1786177063.418788:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418792:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177063.418793:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177063.418795:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00000200:0.0:1786177063.418797:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856332544, offset 0
00000400:00000200:0.0:1786177063.418799:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177063.418802:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177063.418804:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177063.418806:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177063.418808:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177063.418810:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177063.418811:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177063.418813:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254769:254769:256:4294967295] 192.168.202.110@tcp LPNI seq info [254758:254758:8:4294967295]
00000400:00000200:0.0:1786177063.418816:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177063.418820:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177063.418823:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177063.418826:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177063.418828:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177063.418830:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1786177063.418833:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177063.418834:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1786177063.418837:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177063.418839:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177063.418842:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177063.418845:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177063.418848:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:0.0:1786177063.418850:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:0.0:1786177063.418852:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177063.418856:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418864:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177063.418867:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418874:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.418876:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177063.418879:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177063.418882:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177063.418886:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177063.418888:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301959187
00000100:00000001:0.0:1786177063.418890:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301962957
00000100:00000001:0.0:1786177063.418892:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301959187
00000100:00000001:0.0:1786177063.418894:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301962957
00000100:00000001:0.0:1786177063.418896:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177063.418898:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.418900:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177063.418905:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177063/real 1786177063]  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177063.418913:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177063.418915:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177063.418917:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c3c8.
00000400:00000200:0.0:1786177063.418919:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c44c8
00000400:00000010:0.0:1786177063.418921:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c44c8.
00000100:00000001:0.0:1786177063.418924:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177063.418927:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177063.418934:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418938:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177063.418939:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418941:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177063.418942:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177063.418944:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418948:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177063.418951:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418956:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177063.418957:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418958:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177063.418959:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418960:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177063.418961:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177063.418963:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177063.418965:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177063.418966:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177063.418968:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177063.418973:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856332544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177063.418975:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.418977:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177063.418979:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856332416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.418983:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177063.418984:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177063.418986:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177063.418987:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177063.418988:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177063.418989:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177063.418991:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177063.418992:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177063.418994:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177063.418995:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177063.418996:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:0.0:1786177063.418998:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177063.418999:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.419000:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177063.419002:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856332544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177063.419006:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177063.419006:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177063.419007:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786177063.419008:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177063.419009:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177063.419010:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177063.419011:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177063.419012:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786177063.419013:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177063.419013:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177063.419014:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:0.0:1786177063.419015:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177063.419016:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.419017:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177063.419020:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177063.419021:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177063.419022:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177063.419023:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177063.992601:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177063.992607:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177063.992612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177063.992613:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177063.992616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177063.992617:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177063.992662:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177063.992666:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177063.992669:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177064.440593:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177064.440600:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177064.440602:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177064.440607:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177064.440610:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177064.440612:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177064.440614:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177064.440617:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177064.440619:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177065.016620:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177065.016625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177065.016629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177065.016629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177065.016632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177065.016633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177065.016684:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177065.016687:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177065.016689:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177065.464616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177065.464623:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177065.464625:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177065.464629:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177065.464632:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177065.464635:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177065.464637:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177065.464639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177065.464642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177066.040634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177066.040639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177066.040643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177066.040644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177066.040647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177066.040648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177066.040710:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177066.040714:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177066.040717:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177066.488627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177066.488634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177066.488635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177066.488640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177066.488662:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177066.488666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177066.488669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177066.488672:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177066.488674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177067.064623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177067.064629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177067.064633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177067.064634:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177067.064636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177067.064637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177067.064691:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177067.064694:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177067.064697:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177067.256589:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301962957, at 4301963008
00000100:00000040:1.0:1786177067.256589:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301962957, at 4301963008
00000100:00000010:1.0:1786177067.256603:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134780.
00000100:00000010:0.0:1786177067.256605:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:2.0:1786177067.512588:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177067.512595:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177067.512597:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177067.512601:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177067.512605:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177067.512607:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177067.512610:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177067.512612:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177067.512614:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177068.088646:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177068.088651:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177068.088655:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177068.088656:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177068.088659:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177068.088660:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177068.088717:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177068.088720:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177068.088724:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.0:1786177068.408601:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:3.0:1786177068.408612:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:3.0:1786177068.408617:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1786177068.408832:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:1.0:1786177068.408845:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1786177068.408849:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1786177068.408857:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786177068.408861:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177068.536623:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177068.536631:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177068.536633:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177068.536634:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1786177068.536637:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177068.536641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177068.536642:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177068.536647:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177068.536650:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7291
00000100:00100000:3.0:1786177068.536654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000400:00000200:2.0:1786177068.536659:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177068.536661:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000400:00000200:2.0:1786177068.536667:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177068.536667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177068.536672:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177068.536673:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1786177068.536676:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177068.536679:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177068.536680:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177068.536684:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177068.536685:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.536687:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177068.536692:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177068.536694:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177068.536698:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177068.536702:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177068.536706:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177068.536708:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177068.536712:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.536715:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177068.536717:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.536720:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177068.536724:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177068.536728:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177068.536731:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177068.536734:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177068.536737:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177068.536741:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.536743:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177068.536745:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177068.536761:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786177068.536765:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177068.536769:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177068.536771:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.536774:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177068.536777:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177068.536781:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177068.536790:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786177068.536796:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177068.536806:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177068.536816:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177068.536819:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177068.536823:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177068.536844:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177068.536846:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177068.536850:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177068.536852:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177068.536853:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177068.536856:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177068.536859:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7291
00000400:00000200:2.0:1786177068.536863:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177068.536866:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177068.536869:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177068.536872:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.536874:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177068.536877:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.536880:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177068.536883:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:0.0:1786177068.536893:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1786177068.536894:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177068.536899:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786177068.536900:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1786177068.536902:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:0.0:1786177068.536904:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1786177068.536905:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177068.536908:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177068.536913:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.536914:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177068.536915:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177068.536916:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.536920:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177068.536925:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177068.536931:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177068.536939:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.536940:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177068.536944:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177068.536945:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177068.536949:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177068.536953:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177068.536954:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177068.536959:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177068.536960:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.536963:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00100000:0.0:1786177068.536968:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856332672:192.168.202.110@tcp:250:kworker.0
00000008:00100000:2.0:1786177068.536970:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1786177068.536975:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1786177068.536976:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.536979:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177068.536981:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:2.0:1786177068.536983:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
02000000:00000001:0.0:1786177068.536985:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786177068.536987:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177068.536991:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1786177068.536991:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177068.536994:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.536996:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177068.536997:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:0.0:1786177068.536999:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4c00.
02000000:00000001:2.0:1786177068.537000:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177068.537018:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
02000000:00000001:0.0:1786177068.537020:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.537022:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177068.537031:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1786177068.537031:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1786177068.537040:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c3c8.
00000100:00000001:2.0:1786177068.537041:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177068.537046:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177068.537051:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c44c8.
00000100:00000200:0.0:1786177068.537060:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856332672, portal 25
00000100:00000040:2.0:1786177068.537061:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177068.537066:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177068.537069:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:0.0:1786177068.537081:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177068.537081:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177068.537088:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177068.537097:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177068.537100:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177068.537103:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177068.537110:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177068.537115:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177068.537115:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:0.0:1786177068.537119:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4770.
00000100:00080000:2.0:1786177068.537123:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7291
00000100:00000200:0.0:1786177068.537125:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856332672, offset 0
00000400:00000200:2.0:1786177068.537131:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177068.537137:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177068.537140:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786177068.537142:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177068.537145:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.537149:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177068.537152:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177068.537154:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177068.537158:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177068.537162:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:0.0:1786177068.537165:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:2.0:1786177068.537165:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177068.537169:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177068.537173:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177068.537175:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786177068.537177:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786177068.537181:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177068.537183:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786177068.537185:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786177068.537191:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:2.0:1786177068.537192:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.537197:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786177068.537198:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000040:2.0:1786177068.537203:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:0.0:1786177068.537206:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00080000:2.0:1786177068.537211:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1786177068.537216:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254770:254770:256:4294967295] 192.168.202.110@tcp LPNI seq info [254759:254759:8:4294967295]
00000100:00000001:2.0:1786177068.537218:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177068.537223:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786177068.537227:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177068.537228:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177068.537231:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.537233:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177068.537236:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177068.537240:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177068.537244:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786177068.537244:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177068.537249:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786177068.537252:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177068.537256:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:0.0:1786177068.537257:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1786177068.537258:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.537261:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177068.537264:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177068.537265:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1786177068.537267:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
02000000:00000010:2.0:1786177068.537269:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786177068.537274:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177068.537279:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:2.0:1786177068.537280:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1786177068.537292:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177068.537300:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177068.537302:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177068.537305:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177068.537310:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177068.537312:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177068.537314:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177068.537316:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177068.537317:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177068.537317:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177068.537323:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177068.537327:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7291
00000400:00000200:0.0:1786177068.537329:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177068.537333:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177068.537336:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301968077
00000400:00000200:2.0:1786177068.537339:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177068.537343:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177068.537345:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000010:0.0:1786177068.537347:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:2.0:1786177068.537347:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177068.537350:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.537353:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177068.537354:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177068.537355:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:0.0:1786177068.537357:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000001:2.0:1786177068.537357:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177068.537361:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177068.537363:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177068.537365:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:0.0:1786177068.537369:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301968077
00000100:00000040:2.0:1786177068.537369:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.537373:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177068.537375:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.537377:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000010:0.0:1786177068.537378:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134360.
00000020:00000040:2.0:1786177068.537380:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177068.537385:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177068.537386:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786177068.537387:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177068.537391:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177068.537394:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177068.537395:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177068.537396:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.537401:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177068.537402:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1786177068.537403:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786177068.537408:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.537413:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:2.0:1786177068.537416:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.537418:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.537421:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000010:2.0:1786177068.537422:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000100:00000001:0.0:1786177068.537425:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000020:00000040:2.0:1786177068.537428:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:0.0:1786177068.537431:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:2.0:1786177068.537432:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1786177068.537435:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1786177068.537436:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.537439:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786177068.537440:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177068.537444:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177068.537447:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177068.537449:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1786177068.537453:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786177068.537459:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.537466:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177068.537470:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177068.537475:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177068.537477:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1786177068.537479:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1786177068.537480:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.537492:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177068.537496:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177068.537498:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856332800:192.168.202.110@tcp:38:kworker.0
00000100:00000040:2.0:1786177068.537504:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177068.537506:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1786177068.537543:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177068.537545:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:0.0:1786177068.537548:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:2.0:1786177068.537548:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177068.537552:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:0.0:1786177068.537553:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177068.537554:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177068.537559:0:24919:0:(import.c:485:import_select_connection()) Process entered
02000000:00000001:0.0:1786177068.537560:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:2.0:1786177068.537563:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7291
02000000:00000010:0.0:1786177068.537567:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
00000400:00000200:2.0:1786177068.537568:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786177068.537572:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177068.537574:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177068.537578:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786177068.537579:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177068.537583:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000100:00000040:2.0:1786177068.537584:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.537589:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1786177068.537590:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4cc0.
00000100:00000040:2.0:1786177068.537591:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177068.537595:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:0.0:1786177068.537596:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856332800, portal 10
00000020:00000001:2.0:1786177068.537597:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:0.0:1786177068.537599:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000020:00000040:2.0:1786177068.537600:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:0.0:1786177068.537602:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000020:00000001:2.0:1786177068.537603:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177068.537608:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786177068.537611:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786177068.537612:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177068.537616:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177068.537625:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177068.537627:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786177068.537631:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177068.537635:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177068.537637:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:0.0:1786177068.537641:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c42a8.
00000100:00000001:2.0:1786177068.537642:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:0.0:1786177068.537646:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856332800, offset 0
00000020:00000040:2.0:1786177068.537647:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786177068.537654:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1786177068.537659:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177068.537660:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177068.537664:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786177068.537666:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177068.537669:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1786177068.537674:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:2.0:1786177068.537675:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.537680:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:0.0:1786177068.537684:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00100000:2.0:1786177068.537685:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:0.0:1786177068.537690:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00000001:2.0:1786177068.537691:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.537695:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177068.537696:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177068.537702:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000010:2.0:1786177068.537702:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000400:00000200:0.0:1786177068.537709:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254771:254771:255:4294967295] 192.168.202.110@tcp LPNI seq info [254760:254760:7:4294967295]
00000020:00000040:2.0:1786177068.537709:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177068.537715:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177068.537719:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177068.537720:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177068.537724:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177068.537727:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177068.537731:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177068.537734:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000010:2.0:1786177068.537738:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000800:00000200:0.0:1786177068.537743:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177068.537745:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1786177068.537749:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45a00.
00000100:00100000:2.0:1786177068.537754:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1786177068.537760:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177068.537769:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177068.537773:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177068.537774:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177068.537779:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177068.537782:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000100:00000040:2.0:1786177068.537787:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177068.537789:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177068.537794:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301964298
00000100:00000001:0.0:1786177068.537800:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301968077
00000100:00000001:0.0:1786177068.537805:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301964298
00000100:00000001:2.0:1786177068.537806:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.537809:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301968077
00000100:00000001:0.0:1786177068.537814:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177068.537817:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.537819:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177068.537820:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7301)
00000100:00000040:0.0:1786177068.537826:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177068.537835:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177068.537837:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177068.537839:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177068.537842:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177068.537844:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177068.537848:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856332928:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177068.537852:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177068.537855:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177068.537856:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177068.537858:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177068.537862:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4800.
02000000:00000001:0.0:1786177068.537864:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177068.537867:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177068.537870:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786177068.537873:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c47f8.
00000100:00000200:0.0:1786177068.537876:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856332928, portal 10
00000100:00000001:0.0:1786177068.537878:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177068.537879:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:0.0:1786177068.537885:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177068.537891:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177068.537894:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177068.537896:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4198.
00000100:00000200:0.0:1786177068.537898:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856332928, offset 0
00000400:00000200:0.0:1786177068.537903:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177068.537907:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177068.537911:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177068.537913:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177068.537916:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177068.537918:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177068.537920:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177068.537923:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254772:254772:254:4294967295] 192.168.202.110@tcp LPNI seq info [254761:254761:6:4294967295]
00000400:00000200:0.0:1786177068.537927:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177068.537932:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177068.537937:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177068.537940:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45200.
00000800:00000200:0.0:1786177068.537944:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177068.537948:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177068.537951:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177068.537955:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177068.537958:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177068.537961:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301964423
00000100:00000001:0.0:1786177068.537963:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301968077
00000100:00000001:0.0:1786177068.537965:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301964423
00000100:00000001:0.0:1786177068.537967:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301968077
00000100:00000001:0.0:1786177068.537969:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177068.537970:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.537972:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.537974:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177068.537978:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177068.537980:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1786177068.537982:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177068.537983:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177068.537984:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177068.537988:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177068.537995:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177068.537996:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177068.537999:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177068.538001:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177068.538003:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177068.538025:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856333056:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177068.538029:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177068.538031:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177068.538033:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177068.538034:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177068.538039:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786177068.538041:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177068.538045:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177068.538049:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c738.
00000400:00000010:0.0:1786177068.538053:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177068.538056:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856333056, portal 4
00000100:00000001:0.0:1786177068.538059:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177068.538060:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000040:0.0:1786177068.538068:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177068.538075:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177068.538078:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177068.538080:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177068.538083:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856333056, offset 0
00000400:00000200:0.0:1786177068.538086:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177068.538090:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177068.538093:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177068.538096:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177068.538099:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177068.538101:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177068.538103:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177068.538106:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254773:254773:253:4294967295] 192.168.202.110@tcp LPNI seq info [254762:254762:5:4294967295]
00000400:00000200:0.0:1786177068.538109:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177068.538115:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177068.538120:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177068.538123:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000800:00000200:0.0:1786177068.538127:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177068.538131:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177068.538133:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177068.538136:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177068.538140:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177068.538142:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301964298
00000100:00000001:0.0:1786177068.538145:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301968077
00000100:00000001:0.0:1786177068.538147:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301964298
00000100:00000001:0.0:1786177068.538149:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301968077
00000100:00000001:0.0:1786177068.538151:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177068.538154:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.538157:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177068.538161:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177068.538168:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177068.538170:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177068.538172:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177068.538174:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177068.538176:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177068.538179:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856333184:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177068.538183:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177068.538185:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177068.538187:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177068.538189:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177068.538192:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177068.538194:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177068.538197:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177068.538200:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177068.538203:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177068.538206:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856333184, portal 4
00000100:00000001:0.0:1786177068.538208:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177068.538209:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:0.0:1786177068.538215:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177068.538221:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177068.538223:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177068.538226:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4a18.
00000100:00000200:0.0:1786177068.538228:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856333184, offset 0
00000400:00000200:0.0:1786177068.538233:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177068.538237:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177068.538241:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177068.538244:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177068.538246:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177068.538249:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177068.538251:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177068.538254:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254774:254774:252:4294967295] 192.168.202.110@tcp LPNI seq info [254763:254763:4:4294967295]
00000400:00000200:0.0:1786177068.538259:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177068.538266:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177068.538270:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177068.538276:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177068.538280:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177068.538284:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177068.538286:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177068.538290:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177068.538293:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177068.538295:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301964306
00000100:00000001:0.0:1786177068.538298:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301968077
00000100:00000001:0.0:1786177068.538300:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301964306
00000100:00000001:0.0:1786177068.538303:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301968077
00000100:00000001:0.0:1786177068.538306:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177068.538308:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.538311:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.538313:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177068.538316:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177068.538318:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1786177068.538320:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177068.538321:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177068.538323:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177068.538326:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177069.112616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177069.112621:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177069.112625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177069.112626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177069.112629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177069.112630:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177069.112691:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177069.112695:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177069.112698:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177069.560659:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177069.560666:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177069.560669:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177069.560674:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177069.560679:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177069.560683:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177069.560685:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177069.560688:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177069.560691:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177070.136595:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177070.136600:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177070.136604:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177070.136605:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177070.136608:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177070.136609:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177070.136665:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177070.136669:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177070.136673:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177070.584698:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177070.584705:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177070.584708:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177070.584713:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177070.584716:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177070.584720:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177070.584723:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177070.584725:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177070.584728:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177071.160587:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177071.160595:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177071.160599:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177071.160602:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177071.160607:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177071.160609:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177071.160611:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177071.160614:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177071.160618:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177071.160619:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177071.160621:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177071.160623:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177071.160625:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177071.160626:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177071.160628:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177071.160630:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177071.160633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177071.160635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177071.160686:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177071.160689:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177071.160692:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000008:00000020:1.0:1786177071.608578:0:104859:0:(osc_request.c:745:osc_update_next_shrink()) next time 8500 to shrink grant
00000100:00000001:3.0:1786177071.608704:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177071.608708:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177071.608710:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177071.608713:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177071.608716:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177071.608718:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177071.608720:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177071.608722:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177071.608724:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1786177071.608796:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1786177071.608834:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1786177071.608842:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177071.608846:0:2381:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1786177071.608851:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177071.608861:0:2381:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1786177071.608867:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177071.608872:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:1.0:1786177071.608877:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1786177071.608883:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177071.608889:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4770
00000400:00000010:1.0:1786177071.608891:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4770.
00000100:00000001:1.0:1786177071.608897:0:2381:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177071.608906:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177071.608932:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177071.608937:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177071.608945:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177071.608947:0:2381:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1786177071.608957:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177071.608960:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45a00.
00000400:00000200:1.0:1786177071.608964:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1786177071.608968:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177071.608972:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c42a8
00000400:00000010:1.0:1786177071.608987:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c42a8.
00000100:00000001:1.0:1786177071.608991:0:2381:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177071.608996:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609000:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177071.609007:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177071.609008:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177071.609014:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:1.0:1786177071.609014:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177071.609025:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177071.609027:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177068/real 1786177071]  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177071.609029:0:2381:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1786177071.609040:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177071.609046:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45200.
00000400:00000200:1.0:1786177071.609051:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000001:0.0:1786177071.609053:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:1.0:1786177071.609056:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000001:0.0:1786177071.609058:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177071.609066:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177071.609070:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4cc0
00000400:00000010:0.0:1786177071.609072:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4cc0.
00000100:00000001:0.0:1786177071.609076:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000400:00000200:1.0:1786177071.609077:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4198
00000400:00000010:1.0:1786177071.609082:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4198.
00000100:00000200:0.0:1786177071.609084:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177071.609088:0:2381:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177071.609096:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177071.609109:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000200:0.0:1786177071.609114:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786177071.609115:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609141:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177071.609145:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177071.609148:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177071.609150:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177071.609154:0:2381:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177071.609155:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177071.609165:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:1.0:1786177071.609168:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177071.609175:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:1.0:1786177071.609181:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:0.0:1786177071.609183:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:1.0:1786177071.609189:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177071.609194:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000100:00000040:0.0:1786177071.609195:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1786177071.609199:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:1.0:1786177071.609204:0:2381:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1786177071.609210:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000200:1.0:1786177071.609215:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609216:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609220:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177071.609223:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609229:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177071.609235:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177071.609236:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177071.609244:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00000040:1.0:1786177071.609244:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177071.609255:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177071.609259:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177071.609262:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177071.609270:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177071.609270:0:2381:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1786177071.609282:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000100:00100000:0.0:1786177071.609287:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856332800:192.168.202.110@tcp:38:kworker.0
00000800:00000010:1.0:1786177071.609289:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:1.0:1786177071.609295:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:0.0:1786177071.609298:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000400:00000200:1.0:1786177071.609307:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000100:00100000:0.0:1786177071.609312:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177068/real 1786177071]  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786177071.609315:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4a18
00000400:00000010:1.0:1786177071.609319:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4a18.
00000100:00000001:1.0:1786177071.609325:0:2381:0:(events.c:34:request_out_callback()) Process entered
00000400:00000001:0.0:1786177071.609331:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000200:1.0:1786177071.609334:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177071.609335:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177071.609338:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786177071.609345:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c47f8
00000400:00000010:0.0:1786177071.609349:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c47f8.
00000100:00000001:1.0:1786177071.609349:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177071.609354:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:1.0:1786177071.609355:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177071.609363:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177071.609365:0:2381:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000200:0.0:1786177071.609375:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177071.609386:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609393:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177071.609395:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609397:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177071.609398:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177071.609402:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609410:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177071.609414:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609421:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177071.609425:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609427:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177071.609428:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609430:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177071.609432:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177071.609435:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177071.609439:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177071.609441:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177071.609445:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177071.609454:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856332928:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177071.609458:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177071.609461:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177068/real 1786177071]  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177071.609469:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177071.609470:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177071.609472:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c3c8.
00000400:00000200:0.0:1786177071.609475:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c44c8
00000400:00000010:0.0:1786177071.609476:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c44c8.
00000100:00000001:0.0:1786177071.609479:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177071.609482:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177071.609491:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609498:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177071.609500:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609502:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177071.609503:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177071.609508:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609559:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177071.609563:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609570:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177071.609573:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609575:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177071.609576:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609578:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177071.609580:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177071.609583:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177071.609587:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177071.609590:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177071.609594:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177071.609603:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856332672:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177071.609607:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177071.609610:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177071.609613:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856332800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609620:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177071.609624:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177071.609629:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177071.609634:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177071.609636:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177071.609638:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177071.609641:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177071.609644:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:0.0:1786177071.609648:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177071.609651:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177071.609653:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:0.0:1786177071.609657:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177071.609659:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609660:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177071.609664:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856332928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609670:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177071.609671:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177071.609672:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4800.
02000000:00000001:0.0:1786177071.609675:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177071.609676:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177071.609678:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177071.609680:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177071.609682:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0800.
02000000:00000001:0.0:1786177071.609687:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177071.609688:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177071.609690:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:0.0:1786177071.609693:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177071.609694:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609696:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177071.609700:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856332672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609706:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177071.609708:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177071.609709:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177071.609713:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177071.609714:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177071.609715:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177071.609717:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177071.609718:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:0.0:1786177071.609720:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177071.609721:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177071.609722:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:0.0:1786177071.609724:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177071.609725:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609729:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177071.609731:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177071.609733:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177071.609763:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177071.609768:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177068/real 1786177071]  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177071.609776:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177071.609777:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177071.609779:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c738.
00000400:00000200:0.0:1786177071.609783:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177071.609784:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177071.609786:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177071.609789:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177071.609797:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609803:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177071.609804:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609806:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177071.609807:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177071.609810:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609816:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177071.609821:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609828:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177071.609831:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609833:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177071.609835:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609836:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177071.609838:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177071.609841:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177071.609845:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177071.609847:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177071.609851:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177071.609866:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856333056:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177071.609870:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177071.609874:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177068/real 1786177071]  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177071.609882:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177071.609883:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177071.609885:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177071.609888:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177071.609889:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177071.609891:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177071.609895:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177071.609904:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609911:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177071.609912:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609915:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177071.609916:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177071.609920:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609928:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177071.609933:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.609947:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177071.609950:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609952:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177071.609954:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.609956:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177071.609958:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177071.609962:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177071.609966:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177071.609968:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177071.609972:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177071.609993:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856333184:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177071.609998:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.610000:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177071.610003:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856333056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.610009:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177071.610010:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177071.610012:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786177071.610016:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177071.610017:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177071.610019:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177071.610021:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177071.610023:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:0.0:1786177071.610025:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177071.610027:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177071.610028:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:0.0:1786177071.610031:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177071.610032:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.610035:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177071.610037:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856333184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177071.610043:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177071.610044:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177071.610046:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177071.610048:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177071.610049:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177071.610050:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177071.610052:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177071.610053:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:0.0:1786177071.610054:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177071.610055:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177071.610056:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:0.0:1786177071.610059:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177071.610060:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.610062:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177071.610070:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177071.610071:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177071.610073:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177071.610075:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177072.184626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177072.184631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177072.184636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177072.184636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177072.184639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177072.184640:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177072.184694:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177072.184697:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177072.184700:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177072.376614:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301968077, at 4301968128
00000100:00000040:1.0:1786177072.376614:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301968077, at 4301968128
00000100:00000010:1.0:1786177072.376637:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134360.
00000100:00000010:0.0:1786177072.376639:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:3.0:1786177072.632674:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177072.632680:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177072.632682:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177072.632686:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177072.632688:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177072.632692:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177072.632695:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177072.632696:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177072.632699:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177073.208585:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177073.208589:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177073.208593:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177073.208594:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177073.208597:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177073.208600:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177073.208601:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177073.208604:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177073.208606:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177073.208608:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177073.208609:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177073.208611:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177073.208671:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177073.208674:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177073.208677:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1786177073.656647:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177073.656656:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1786177073.656658:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177073.656661:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177073.656665:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177073.656672:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7296
00000400:00000200:1.0:1786177073.656680:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177073.656687:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177073.656689:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177073.656691:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000040:1.0:1786177073.656692:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.656697:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177073.656697:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000040:1.0:1786177073.656700:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177073.656702:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.656703:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177073.656707:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177073.656709:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:1.0:1786177073.656711:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177073.656713:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:1.0:1786177073.656715:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:1.0:1786177073.656717:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:1.0:1786177073.656721:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00100000:3.0:1786177073.656722:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786177073.656723:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177073.656726:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177073.656729:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177073.656730:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177073.656734:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00000001:3.0:1786177073.656735:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:1.0:1786177073.656738:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177073.656741:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177073.656742:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177073.656747:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177073.656748:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1786177073.656750:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177073.656754:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177073.656755:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.656762:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177073.656764:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177073.656771:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa7480.
00000020:00000040:1.0:1786177073.656775:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1786177073.656780:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177073.656782:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.656786:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177073.656789:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177073.656803:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177073.656811:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b00c00.
00000100:00000001:1.0:1786177073.656818:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177073.656828:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177073.656839:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177073.656844:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177073.656849:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000492901e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177073.656880:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177073.656883:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177073.656886:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1786177073.656887:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177073.656889:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177073.656892:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177073.656895:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7296
00000400:00000200:1.0:1786177073.656900:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177073.656903:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177073.656905:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177073.656908:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.656910:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177073.656912:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.656914:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177073.656916:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177073.656923:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:1.0:1786177073.656925:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177073.656927:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:1.0:1786177073.656929:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:1.0:1786177073.656931:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:1.0:1786177073.656934:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.656936:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177073.656938:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.656940:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177073.656943:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:1.0:1786177073.656946:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177073.656948:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177073.656950:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177073.656990:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177073.656993:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.656997:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177073.657002:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1786177073.657005:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.657008:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177073.657013:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa4000.
00000020:00000040:1.0:1786177073.657016:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:0.0:1786177073.657018:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:1.0:1786177073.657019:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177073.657023:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.657024:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177073.657026:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786177073.657028:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:1.0:1786177073.657029:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177073.657034:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177073.657039:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b03000.
00000100:00000040:0.0:1786177073.657041:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177073.657045:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.657056:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00100000:1.0:1786177073.657058:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177073.657060:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177073.657067:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177073.657071:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177073.657073:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:0.0:1786177073.657076:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.657079:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177073.657085:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856333312:192.168.202.110@tcp:250:kworker.0
00000100:00000040:1.0:1786177073.657088:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd947f83] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.657093:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177073.657099:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1786177073.657103:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177073.657104:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177073.657107:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:0.0:1786177073.657109:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:1.0:1786177073.657115:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
02000000:00000010:0.0:1786177073.657117:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5400.
00000100:00000001:1.0:1786177073.657118:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:0.0:1786177073.657122:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177073.657123:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177073.657129:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1786177073.657134:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1786177073.657135:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7296
00000400:00000010:0.0:1786177073.657144:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000200:1.0:1786177073.657144:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177073.657152:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000010:0.0:1786177073.657155:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00080000:1.0:1786177073.657155:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177073.657165:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.657167:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177073.657171:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.657175:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177073.657177:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000200:0.0:1786177073.657179:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856333312, portal 25
00000020:00000040:1.0:1786177073.657180:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000100:00000001:0.0:1786177073.657185:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000020:00000001:1.0:1786177073.657185:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786177073.657189:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751755904 : -114547957795712 : ffff97d1b9aa7480)
00000020:00000040:1.0:1786177073.657190:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:1.0:1786177073.657196:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:1.0:1786177073.657201:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177073.657202:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786177073.657207:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.657215:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177073.657220:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177073.657221:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1786177073.657225:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:0.0:1786177073.657229:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000040:1.0:1786177073.657233:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000010:0.0:1786177073.657235:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00080000:1.0:1786177073.657241:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:0.0:1786177073.657242:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856333312, offset 0
00000100:00000001:1.0:1786177073.657246:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177073.657250:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786177073.657254:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:1.0:1786177073.657255:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177073.657259:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.657261:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177073.657264:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1786177073.657265:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:1.0:1786177073.657270:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.657274:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177073.657277:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000010:1.0:1786177073.657282:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa4700.
00000400:00000200:0.0:1786177073.657285:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:1.0:1786177073.657287:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:0.0:1786177073.657292:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:1.0:1786177073.657293:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177073.657296:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.657299:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1786177073.657300:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:1.0:1786177073.657302:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1786177073.657306:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:1.0:1786177073.657306:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177073.657315:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254775:254775:256:4294967295] 192.168.202.110@tcp LPNI seq info [254764:254764:8:4294967295]
02000000:00000010:1.0:1786177073.657315:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b01800.
00000100:00000001:1.0:1786177073.657323:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177073.657325:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:1.0:1786177073.657334:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1786177073.657344:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177073.657351:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177073.657360:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000100:00000001:1.0:1786177073.657361:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177073.657367:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177073.657371:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1786177073.657374:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c2491d39] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1786177073.657377:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786177073.657389:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1786177073.657392:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786177073.657394:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000100:00080000:1.0:1786177073.657397:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000100:0.0:1786177073.657404:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:1.0:1786177073.657406:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000010:0.0:1786177073.657408:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000100:00000001:1.0:1786177073.657409:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177073.657413:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1786177073.657417:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1786177073.657419:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177073.657426:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7296
00000400:00000200:0.0:1786177073.657428:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177073.657436:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177073.657441:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177073.657443:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177073.657447:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.657449:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177073.657451:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.657453:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177073.657456:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177073.657458:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:1.0:1786177073.657460:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177073.657463:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:1.0:1786177073.657465:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:1.0:1786177073.657468:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:1.0:1786177073.657471:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.657474:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177073.657475:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.657478:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177073.657480:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:1.0:1786177073.657483:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177073.657486:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177073.657487:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177073.657491:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177073.657493:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.657495:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177073.657497:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1786177073.657500:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.657502:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177073.657507:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa5f80.
00000020:00000040:1.0:1786177073.657540:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:1.0:1786177073.657544:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177073.657546:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.657548:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177073.657550:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177073.657552:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177073.657557:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b02400.
00000100:00000001:1.0:1786177073.657562:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177073.657567:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177073.657574:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177073.657577:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177073.657581:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5f3cd3c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177073.657590:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177073.657593:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177073.657597:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1786177073.657598:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177073.657600:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177073.657602:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177073.657605:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7296
00000400:00000200:0.0:1786177073.658574:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000200:1.0:1786177073.658577:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177073.658580:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177073.658586:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:1.0:1786177073.658586:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177073.658592:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:0.0:1786177073.658595:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786177073.658601:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.658608:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177073.658611:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177073.658613:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:0.0:1786177073.658619:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177073.658619:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177073.658627:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177073.658630:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:1.0:1786177073.658635:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786177073.658636:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.658640:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:1.0:1786177073.658640:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:0.0:1786177073.658650:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1786177073.658656:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:0.0:1786177073.658658:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:1.0:1786177073.658662:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:0.0:1786177073.658665:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301973199
00000100:00000040:1.0:1786177073.658671:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.658675:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:0.0:1786177073.658679:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000040:1.0:1786177073.658679:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177073.658685:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177073.658689:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:0.0:1786177073.658690:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:1.0:1786177073.658694:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177073.658700:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301973199
00000100:00000001:1.0:1786177073.658700:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177073.658708:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1851345a0.
02000000:00000001:1.0:1786177073.658708:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177073.658714:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177073.658715:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:1.0:1786177073.658720:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.658723:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177073.658725:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177073.658728:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1786177073.658730:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786177073.658734:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000008:00000001:1.0:1786177073.658735:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.658739:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177073.658746:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa7800.
00000020:00000040:1.0:1786177073.658749:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:1.0:1786177073.658752:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177073.658754:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177073.658756:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177073.658758:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177073.658760:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177073.658766:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b03c00.
00000100:00000001:1.0:1786177073.658771:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177073.658778:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177073.658787:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177073.658789:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177073.658794:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e989e18] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177073.658802:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177073.658806:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7307)
00000100:00100000:0.0:1786177073.658820:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177073/real 1786177073]  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177073.658834:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177073.658837:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177073.658840:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177073.658843:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177073.658845:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177073.658848:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177073.658854:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177073.658864:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.658872:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177073.658874:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.658876:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177073.658879:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177073.658884:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.658891:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177073.658896:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.658915:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177073.658923:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.658925:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177073.658926:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.658929:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177073.658931:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177073.658935:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177073.658939:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177073.658942:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177073.658946:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177073.658968:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856333312:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177073.658975:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.658978:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177073.658982:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa7480 x1872934856333312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.658989:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177073.658992:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177073.658995:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177073.658999:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177073.659001:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177073.659004:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177073.659006:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177073.659009:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00c00.
02000000:00000001:0.0:1786177073.659014:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177073.659015:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177073.659017:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa7480.
00000100:00000001:0.0:1786177073.659020:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177073.659021:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.659024:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.659031:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177073.659033:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.659036:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177073.659038:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177073.659040:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177073.659045:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659053:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177073.659054:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177073.659057:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177073.659059:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177073.659062:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177073.659065:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856333440:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177073.659068:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177073.659070:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177073.659072:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177073.659075:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177073.659079:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177073.659082:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177073.659087:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177073.659090:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177073.659094:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177073.659099:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856333440, portal 10
00000100:00000001:0.0:1786177073.659101:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177073.659102:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751742464 : -114547957809152 : ffff97d1b9aa4000)
00000100:00000040:0.0:1786177073.659109:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659116:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177073.659118:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177073.659120:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177073.659123:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856333440, offset 0
00000400:00000200:0.0:1786177073.659128:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177073.659135:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177073.659141:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177073.659146:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177073.659149:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177073.659153:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177073.659156:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177073.659160:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254776:254776:256:4294967295] 192.168.202.110@tcp LPNI seq info [254765:254765:8:4294967295]
00000400:00000200:0.0:1786177073.659164:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177073.659173:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177073.659178:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177073.659184:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177073.659189:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177073.659192:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786177073.659197:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177073.659201:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1786177073.659205:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177073.659207:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177073.659211:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177073.659215:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177073.659219:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177073.659221:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177073.659223:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177073.659227:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659234:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177073.659237:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659243:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.659244:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177073.659248:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177073.659252:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177073.659255:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177073.659257:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301969419
00000100:00000001:0.0:1786177073.659260:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301973199
00000100:00000001:0.0:1786177073.659262:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301969419
00000100:00000001:0.0:1786177073.659264:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301973199
00000100:00000001:0.0:1786177073.659266:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177073.659268:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.659270:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177073.659273:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177073/real 1786177073]  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177073.659282:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177073.659283:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177073.659285:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177073.659288:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177073.659290:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177073.659292:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177073.659295:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177073.659305:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659312:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177073.659313:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.659315:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177073.659317:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177073.659321:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659327:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177073.659331:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659338:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177073.659341:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.659343:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177073.659344:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.659346:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177073.659348:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177073.659351:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177073.659355:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177073.659357:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177073.659361:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177073.659369:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856333440:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177073.659373:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177073.659377:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659390:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177073.659392:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177073.659394:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177073.659396:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177073.659399:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177073.659402:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856333568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177073.659406:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177073.659408:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177073.659410:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177073.659412:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177073.659415:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786177073.659417:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177073.659423:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177073.659425:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177073.659428:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177073.659431:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856333568, portal 10
00000100:00000001:0.0:1786177073.659433:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177073.659434:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751744256 : -114547957807360 : ffff97d1b9aa4700)
00000100:00000040:0.0:1786177073.659440:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659446:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177073.659448:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177073.659451:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177073.659453:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856333568, offset 0
00000400:00000200:0.0:1786177073.659457:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177073.659463:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177073.659466:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177073.659469:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177073.659472:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177073.659474:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177073.659476:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177073.659480:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254777:254777:256:4294967295] 192.168.202.110@tcp LPNI seq info [254766:254766:8:4294967295]
00000400:00000200:0.0:1786177073.659484:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177073.659491:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177073.659496:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177073.659500:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177073.659503:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177073.659506:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786177073.659534:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177073.659537:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1786177073.659541:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177073.659543:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177073.659547:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177073.659550:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177073.659553:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177073.659555:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177073.659558:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177073.659562:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659569:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177073.659572:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659578:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.659579:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177073.659582:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177073.659585:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177073.659588:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177073.659590:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301969545
00000100:00000001:0.0:1786177073.659592:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301973199
00000100:00000001:0.0:1786177073.659594:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301969545
00000100:00000001:0.0:1786177073.659596:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301973199
00000100:00000001:0.0:1786177073.659597:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177073.659600:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.659601:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177073.659605:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177073/real 1786177073]  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177073.659613:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177073.659614:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177073.659616:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177073.659618:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177073.659619:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177073.659621:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177073.659624:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177073.659632:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659639:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177073.659640:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.659642:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177073.659643:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177073.659647:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659654:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177073.659658:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659665:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177073.659667:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.659669:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177073.659670:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.659672:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177073.659674:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177073.659677:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177073.659682:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177073.659684:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177073.659688:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177073.659696:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856333568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177073.659701:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177073.659705:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659712:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177073.659714:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177073.659716:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177073.659718:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177073.659720:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177073.659723:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856333696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177073.659725:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177073.659727:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177073.659728:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177073.659729:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177073.659732:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177073.659733:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177073.659736:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177073.659738:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177073.659740:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177073.659753:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856333696, portal 4
00000100:00000001:0.0:1786177073.659755:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177073.659757:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751750528 : -114547957801088 : ffff97d1b9aa5f80)
00000100:00000040:0.0:1786177073.659767:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659774:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177073.659777:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177073.659780:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177073.659783:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856333696, offset 0
00000400:00000200:0.0:1786177073.659787:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177073.659792:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177073.659796:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177073.659799:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177073.659802:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177073.659804:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177073.659806:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177073.659810:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254778:254778:256:4294967295] 192.168.202.110@tcp LPNI seq info [254767:254767:8:4294967295]
00000400:00000200:0.0:1786177073.659814:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177073.659820:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177073.659825:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177073.659829:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177073.659833:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177073.659836:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786177073.659840:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177073.659843:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1786177073.659847:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177073.659849:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177073.659852:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177073.659856:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177073.659859:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177073.659861:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177073.659864:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177073.659868:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659876:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177073.659879:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659887:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.659888:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177073.659892:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177073.659895:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177073.659898:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177073.659901:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301969420
00000100:00000001:0.0:1786177073.659903:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301973199
00000100:00000001:0.0:1786177073.659906:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301969420
00000100:00000001:0.0:1786177073.659908:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301973199
00000100:00000001:0.0:1786177073.659910:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177073.659912:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.659914:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177073.659918:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177073/real 1786177073]  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177073.659927:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177073.659929:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177073.659931:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177073.659933:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177073.659934:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177073.659936:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177073.659939:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177073.659944:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659949:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177073.659950:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.659951:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177073.659952:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177073.659954:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659977:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177073.659985:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.659993:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177073.659996:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.659997:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177073.659999:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.660001:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177073.660002:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177073.660005:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177073.660009:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177073.660011:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177073.660016:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177073.660025:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856333696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177073.660029:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177073.660032:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.660039:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177073.660040:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177073.660042:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177073.660044:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177073.660046:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177073.660049:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856333824:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177073.660053:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177073.660055:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177073.660057:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177073.660058:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177073.660062:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4800.
02000000:00000001:0.0:1786177073.660064:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177073.660068:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177073.660070:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177073.660073:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177073.660075:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856333824, portal 4
00000100:00000001:0.0:1786177073.660076:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177073.660077:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751756800 : -114547957794816 : ffff97d1b9aa7800)
00000100:00000040:0.0:1786177073.660082:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.660086:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177073.660088:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177073.660089:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177073.660090:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856333824, offset 0
00000400:00000200:0.0:1786177073.660093:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177073.660096:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177073.660098:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177073.660100:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177073.660102:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177073.660104:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177073.660105:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177073.660108:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254779:254779:256:4294967295] 192.168.202.110@tcp LPNI seq info [254768:254768:8:4294967295]
00000400:00000200:0.0:1786177073.660111:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177073.660117:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177073.660122:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177073.660125:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177073.660129:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177073.660132:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1786177073.660136:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177073.660139:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1786177073.660143:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177073.660145:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177073.660148:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177073.660152:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177073.660155:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177073.660157:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177073.660161:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177073.660165:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.660173:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177073.660177:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.660184:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.660185:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177073.660189:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177073.660192:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177073.660195:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177073.660197:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301969428
00000100:00000001:0.0:1786177073.660200:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301973199
00000100:00000001:0.0:1786177073.660202:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301969428
00000100:00000001:0.0:1786177073.660204:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301973199
00000100:00000001:0.0:1786177073.660206:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177073.660208:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.660210:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177073.660215:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177073/real 1786177073]  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177073.660223:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177073.660225:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177073.660226:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177073.660229:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177073.660231:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177073.660233:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177073.660236:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177073.660242:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.660250:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177073.660251:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.660253:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177073.660254:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177073.660256:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.660260:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177073.660263:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.660267:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177073.660268:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.660269:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177073.660270:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.660271:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177073.660272:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177073.660274:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177073.660276:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177073.660277:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177073.660280:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177073.660285:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856333824:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177073.660287:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.660289:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177073.660291:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4000 x1872934856333440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.660296:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177073.660297:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177073.660299:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177073.660301:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177073.660302:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177073.660304:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177073.660306:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177073.660307:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03000.
02000000:00000001:0.0:1786177073.660309:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177073.660310:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177073.660311:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4000.
00000100:00000001:0.0:1786177073.660313:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177073.660314:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.660315:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177073.660318:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4700 x1872934856333568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.660322:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177073.660323:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177073.660324:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786177073.660325:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177073.660326:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177073.660326:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177073.660327:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177073.660328:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b01800.
02000000:00000001:0.0:1786177073.660331:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177073.660331:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177073.660332:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4700.
00000100:00000001:0.0:1786177073.660333:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177073.660334:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.660335:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177073.660337:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5f80 x1872934856333696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.660341:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177073.660342:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177073.660343:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177073.660344:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177073.660345:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177073.660346:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177073.660347:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177073.660348:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b02400.
02000000:00000001:0.0:1786177073.660350:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177073.660351:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177073.660352:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5f80.
00000100:00000001:0.0:1786177073.660354:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177073.660355:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.660356:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177073.660357:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa7800 x1872934856333824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177073.660361:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177073.660362:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177073.660363:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4800.
02000000:00000001:0.0:1786177073.660364:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177073.660364:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177073.660365:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177073.660366:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177073.660367:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03c00.
02000000:00000001:0.0:1786177073.660368:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177073.660369:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177073.660369:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa7800.
00000100:00000001:0.0:1786177073.660371:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177073.660372:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.660373:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177073.660376:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177073.660377:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177073.660378:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177073.660380:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177074.232644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177074.232652:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177074.232659:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177074.232661:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177074.232664:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177074.232666:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177074.232727:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177074.232732:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177074.232736:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177074.680720:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177074.680726:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177074.680727:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177074.680731:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177074.680733:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177074.680736:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177074.680738:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177074.680741:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177074.680743:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177075.256627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177075.256639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177075.256647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177075.256648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177075.256651:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177075.256652:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177075.256701:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177075.256704:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177075.256707:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177075.704627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177075.704635:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177075.704637:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177075.704644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177075.704648:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177075.704652:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177075.704655:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177075.704658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177075.704662:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177076.280627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177076.280632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177076.280636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177076.280637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177076.280639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177076.280640:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177076.280690:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177076.280694:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177076.280696:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177076.728620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177076.728627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177076.728628:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177076.728633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177076.728636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177076.728639:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177076.728642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177076.728644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177076.728647:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177077.304681:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177077.304689:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177077.304696:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177077.304698:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177077.304703:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177077.304704:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177077.304753:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177077.304758:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177077.304762:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177077.496613:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301973199, at 4301973248
00000100:00000040:1.0:1786177077.496614:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301973199, at 4301973248
00000100:00000010:0.0:1786177077.496631:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000010:1.0:1786177077.496634:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1851345a0.
00000100:00000001:3.0:1786177077.752625:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177077.752635:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177077.752637:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177077.752644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177077.752648:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177077.752652:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177077.752655:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177077.752658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177077.752661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177078.328622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177078.328628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177078.328632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177078.328633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177078.328636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177078.328637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177078.328695:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177078.328698:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177078.328701:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177078.776616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177078.776622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177078.776624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177078.776627:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1786177078.776629:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177078.776633:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1786177078.776637:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177078.776637:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177078.776640:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177078.776641:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177078.776643:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177078.776645:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177078.776647:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177078.776650:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177078.776657:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7302
00000400:00000200:2.0:1786177078.776665:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177078.776671:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177078.776674:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177078.776676:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.776679:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177078.776681:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.776683:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177078.776685:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177078.776687:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177078.776689:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177078.776692:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177078.776693:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177078.776695:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177078.776697:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.776699:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177078.776700:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.776701:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177078.776703:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177078.776705:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177078.776707:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177078.776709:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177078.776711:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177078.776713:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177078.776715:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177078.776716:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177078.776724:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786177078.776726:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177078.776730:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177078.776731:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177078.776734:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177078.776736:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177078.776739:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177078.776745:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786177078.776750:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177078.776757:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177078.776764:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177078.776768:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177078.776771:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177078.776787:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177078.776790:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177078.776792:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177078.776797:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177078.776798:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177078.776800:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177078.776802:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7302
00000400:00000200:2.0:1786177078.776804:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177078.776806:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177078.776808:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177078.776810:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.776812:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177078.776814:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.776815:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177078.776817:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177078.776818:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177078.776819:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177078.776821:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177078.776823:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177078.776825:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177078.776828:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.776829:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177078.776831:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.776832:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177078.776833:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177078.776839:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1786177078.776840:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:0.0:1786177078.776844:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1786177078.776844:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177078.776849:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177078.776854:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1786177078.776856:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177078.776859:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177078.776863:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177078.776868:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177078.776871:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1786177078.776874:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000008:00000001:2.0:1786177078.776876:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.776877:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177078.776879:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.776883:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000010:2.0:1786177078.776884:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
02000000:00000001:0.0:1786177078.776888:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1786177078.776888:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177078.776890:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177078.776892:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.776894:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177078.776895:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177078.776897:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177078.776900:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177078.776903:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856333952:192.168.202.110@tcp:250:kworker.0
02000000:00000010:2.0:1786177078.776904:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:0.0:1786177078.776924:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177078.776928:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1786177078.776930:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.776933:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.776938:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:2.0:1786177078.776938:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177078.776946:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000010:0.0:1786177078.776947:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4c00.
00000100:00000001:2.0:1786177078.776948:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.776952:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177078.776955:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177078.776964:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177078.776971:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000100:00000001:2.0:1786177078.776973:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177078.776977:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177078.776979:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177078.776980:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:0.0:1786177078.776981:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00080000:2.0:1786177078.776982:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177078.776988:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177078.776990:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7302
00000100:00000200:0.0:1786177078.776993:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856333952, portal 25
00000400:00000200:2.0:1786177078.776993:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177078.776999:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1786177078.777000:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1786177078.777001:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177078.777005:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187904 : -114548255363712 : ffff97d1a7eded80)
00000100:00000040:2.0:1786177078.777006:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777011:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177078.777013:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777016:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:0.0:1786177078.777019:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177078.777019:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177078.777021:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177078.777024:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177078.777026:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177078.777030:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786177078.777033:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786177078.777039:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177078.777040:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786177078.777041:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177078.777046:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:0.0:1786177078.777049:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777051:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000010:0.0:1786177078.777053:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000020:00000040:2.0:1786177078.777056:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000200:0.0:1786177078.777058:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856333952, offset 0
00000100:00080000:2.0:1786177078.777063:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1786177078.777070:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777075:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177078.777081:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177078.777082:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786177078.777090:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:2.0:1786177078.777092:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177078.777096:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177078.777099:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786177078.777100:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:0.0:1786177078.777107:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177078.777113:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000008:00100000:2.0:1786177078.777113:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1786177078.777118:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000008:00000001:2.0:1786177078.777123:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177078.777124:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254780:254780:256:4294967295] 192.168.202.110@tcp LPNI seq info [254769:254769:8:4294967295]
00000100:00000001:2.0:1786177078.777128:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177078.777133:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:2.0:1786177078.777136:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000020:00000040:2.0:1786177078.777141:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177078.777147:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:0.0:1786177078.777148:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1786177078.777150:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177078.777155:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177078.777157:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:0.0:1786177078.777159:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:2.0:1786177078.777160:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177078.777167:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000800:00000010:0.0:1786177078.777168:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000100:00000001:2.0:1786177078.777170:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177078.777178:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1786177078.777180:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177078.777187:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1786177078.777194:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000200:0.1:1786177078.777199:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177078.777200:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786177078.777205:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:2.0:1786177078.777211:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:0.0:1786177078.777215:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177078.777219:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000100:00000001:2.0:1786177078.777227:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177078.777228:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:2.0:1786177078.777229:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177078.777234:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:0.0:1786177078.777237:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1786177078.777237:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177078.777240:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1786177078.777244:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000100:00000001:2.0:1786177078.777244:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177078.777248:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7302
00000400:00000010:0.0:1786177078.777250:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000400:00000200:2.0:1786177078.777254:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177078.777258:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:2.0:1786177078.777259:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177078.777261:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:0.0:1786177078.777266:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177078.777267:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777271:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177078.777275:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777279:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177078.777284:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177078.777286:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177078.777289:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786177078.777291:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000040:2.0:1786177078.777291:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177078.777293:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177078.777296:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177078.777299:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177078.777300:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777305:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177078.777308:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777312:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177078.777316:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177078.777320:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177078.777323:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177078.777323:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.777329:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1786177078.777329:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177078.777335:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177078.777336:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177078.777338:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177078.777342:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:0.0:1786177078.777344:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:2.0:1786177078.777346:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786177078.777349:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301978317
00000008:00000001:2.0:1786177078.777351:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177078.777355:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177078.777356:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000010:2.0:1786177078.777360:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000100:00000001:0.0:1786177078.777364:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000040:2.0:1786177078.777365:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177078.777371:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177078.777373:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.777374:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301978317
00000100:00000001:2.0:1786177078.777376:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177078.777379:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000010:0.0:1786177078.777381:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134300.
02000000:00000001:2.0:1786177078.777381:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.777388:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000010:2.0:1786177078.777389:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:0.0:1786177078.777394:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177078.777395:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.777402:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177078.777405:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177078.777406:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1786177078.777414:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00100000:0.0:1786177078.777416:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177078/real 1786177078]  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177078.777418:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177078.777425:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177078.777442:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786177078.777442:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1786177078.777447:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:2.0:1786177078.777447:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1786177078.777452:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000100:00080000:2.0:1786177078.777452:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177078.777455:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177078.777458:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1786177078.777459:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000100:00000001:2.0:1786177078.777460:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:0.0:1786177078.777463:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00080000:2.0:1786177078.777464:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7302
00000100:00000001:0.0:1786177078.777470:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000400:00000200:2.0:1786177078.777471:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177078.777476:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177078.777479:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:0.0:1786177078.777480:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177078.777483:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777487:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177078.777491:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777494:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177078.777498:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177078.777502:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177078.777504:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177078.777506:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786177078.777507:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786177078.777509:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177078.777537:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777540:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177078.777543:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777546:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177078.777549:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000200:0.0:1786177078.777550:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786177078.777555:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177078.777562:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177078.777568:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177078.777569:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177078.777573:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786177078.777573:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177078.777578:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.777580:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177078.777581:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177078.777585:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:2.0:1786177078.777585:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177078.777590:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177078.777593:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177078.777597:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786177078.777598:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786177078.777602:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177078.777606:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177078.777608:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177078.777612:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177078.777614:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1786177078.777616:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786177078.777617:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177078.777623:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000040:0.0:1786177078.777625:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177078.777627:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177078.777635:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177078.777638:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177078.777645:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177078.777649:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177078.777650:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177078.777653:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.777654:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177078.777658:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.777660:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177078.777663:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177078.777669:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:2.0:1786177078.777671:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177078.777677:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000040:2.0:1786177078.777677:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7312)
00000100:00000001:0.0:1786177078.777683:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177078.777688:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177078.777696:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856333952:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177078.777700:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.777702:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177078.777705:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eded80 x1872934856333952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.777711:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177078.777714:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177078.777716:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177078.777719:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177078.777721:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177078.777723:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177078.777725:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177078.777727:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:0.0:1786177078.777730:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177078.777732:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177078.777734:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eded80.
00000100:00000001:0.0:1786177078.777737:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177078.777738:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.777740:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.777747:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177078.777749:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.777751:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177078.777752:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177078.777754:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177078.777757:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.777771:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177078.777773:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.777775:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177078.777777:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177078.777779:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177078.777782:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177078.777785:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177078.777787:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177078.777789:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.777790:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177078.777793:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177078.777795:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177078.777799:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177078.777802:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177078.777805:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177078.777808:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856334080, portal 10
00000100:00000001:0.0:1786177078.777810:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177078.777811:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454180736 : -114548255370880 : ffff97d1a7edd180)
00000100:00000040:0.0:1786177078.777817:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.777823:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177078.777826:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177078.777828:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177078.777830:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856334080, offset 0
00000400:00000200:0.0:1786177078.777834:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177078.777838:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177078.777841:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177078.777844:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177078.777846:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177078.777849:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177078.777851:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177078.777854:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254781:254781:256:4294967295] 192.168.202.110@tcp LPNI seq info [254770:254770:8:4294967295]
00000400:00000200:0.0:1786177078.777858:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177078.777864:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177078.777869:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177078.777871:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177078.777875:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177078.777878:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786177078.777882:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177078.777884:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786177078.777888:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177078.777890:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177078.777892:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177078.777896:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177078.777898:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177078.777900:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177078.777902:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177078.777920:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.777928:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177078.777932:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.777938:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.777940:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177078.777943:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177078.777947:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177078.777951:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177078.777953:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301974538
00000100:00000001:0.0:1786177078.777955:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301978317
00000100:00000001:0.0:1786177078.777957:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301974538
00000100:00000001:0.0:1786177078.777959:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301978317
00000100:00000001:0.0:1786177078.777961:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177078.777964:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.777966:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177078.777969:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177078/real 1786177078]  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177078.777977:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177078.777978:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177078.777980:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177078.777982:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177078.777983:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177078.777985:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177078.777989:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177078.777997:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778003:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177078.778004:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778005:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177078.778007:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177078.778010:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778016:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177078.778020:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778027:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177078.778029:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778030:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177078.778032:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778033:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177078.778035:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177078.778038:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177078.778041:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177078.778043:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177078.778047:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177078.778055:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177078.778058:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177078.778061:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778067:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177078.778068:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.778070:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177078.778072:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177078.778074:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177078.778076:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334208:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177078.778080:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177078.778081:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177078.778083:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.778085:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177078.778088:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4800.
02000000:00000001:0.0:1786177078.778090:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177078.778093:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177078.778095:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177078.778097:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177078.778100:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856334208, portal 10
00000100:00000001:0.0:1786177078.778102:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177078.778103:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178048 : -114548255373568 : ffff97d1a7edc700)
00000100:00000040:0.0:1786177078.778108:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778114:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177078.778117:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177078.778119:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177078.778121:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856334208, offset 0
00000400:00000200:0.0:1786177078.778124:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177078.778128:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177078.778131:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177078.778133:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177078.778136:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177078.778138:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177078.778140:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177078.778143:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254782:254782:256:4294967295] 192.168.202.110@tcp LPNI seq info [254771:254771:8:4294967295]
00000400:00000200:0.0:1786177078.778147:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177078.778153:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177078.778157:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177078.778160:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177078.778163:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177078.778166:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786177078.778169:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177078.778172:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786177078.778175:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177078.778177:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177078.778180:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177078.778183:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177078.778186:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177078.778187:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177078.778189:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177078.778193:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778200:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177078.778203:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778209:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.778211:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177078.778214:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177078.778216:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177078.778219:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177078.778221:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301974663
00000100:00000001:0.0:1786177078.778224:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301978317
00000100:00000001:0.0:1786177078.778227:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301974663
00000100:00000001:0.0:1786177078.778229:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301978317
00000100:00000001:0.0:1786177078.778231:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177078.778232:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.778234:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177078.778237:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177078/real 1786177078]  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177078.778244:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177078.778246:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177078.778247:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177078.778249:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177078.778251:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177078.778253:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177078.778256:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177078.778263:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778269:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177078.778270:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778272:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177078.778273:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177078.778276:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778283:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177078.778286:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778293:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177078.778295:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778296:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177078.778298:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778299:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177078.778301:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177078.778303:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177078.778306:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177078.778308:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177078.778312:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177078.778319:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334208:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177078.778322:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177078.778325:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778331:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177078.778332:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.778334:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177078.778336:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177078.778338:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177078.778340:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334336:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177078.778343:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177078.778345:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177078.778346:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.778348:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177078.778351:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786177078.778352:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177078.778356:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177078.778358:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177078.778361:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177078.778363:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856334336, portal 4
00000100:00000001:0.0:1786177078.778365:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177078.778367:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454181632 : -114548255369984 : ffff97d1a7edd500)
00000100:00000040:0.0:1786177078.778372:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778378:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177078.778380:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177078.778382:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177078.778384:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856334336, offset 0
00000400:00000200:0.0:1786177078.778388:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177078.778391:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177078.778394:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177078.778397:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177078.778399:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177078.778401:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177078.778403:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177078.778406:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254783:254783:256:4294967295] 192.168.202.110@tcp LPNI seq info [254772:254772:8:4294967295]
00000400:00000200:0.0:1786177078.778410:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177078.778416:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177078.778420:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177078.778423:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177078.778426:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177078.778428:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786177078.778432:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177078.778435:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786177078.778438:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177078.778440:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177078.778443:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177078.778446:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177078.778448:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177078.778451:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177078.778453:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177078.778456:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778463:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177078.778466:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778473:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.778474:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177078.778476:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177078.778479:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177078.778482:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177078.778484:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301974538
00000100:00000001:0.0:1786177078.778486:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301978317
00000100:00000001:0.0:1786177078.778488:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301974538
00000100:00000001:0.0:1786177078.778490:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301978317
00000100:00000001:0.0:1786177078.778492:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177078.778494:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.778495:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177078.778499:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177078/real 1786177078]  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177078.778505:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177078.778507:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177078.778508:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177078.778528:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177078.778530:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177078.778532:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177078.778535:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177078.778543:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778550:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177078.778551:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778553:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177078.778554:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177078.778558:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778564:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177078.778567:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778574:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177078.778576:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778578:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177078.778579:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778581:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177078.778582:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177078.778584:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177078.778587:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177078.778590:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177078.778593:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177078.778600:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334336:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177078.778604:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177078.778607:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778613:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177078.778615:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.778617:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177078.778619:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177078.778621:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177078.778624:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177078.778627:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177078.778628:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177078.778630:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177078.778631:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177078.778634:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177078.778636:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177078.778639:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177078.778641:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177078.778644:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177078.778647:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856334464, portal 4
00000100:00000001:0.0:1786177078.778648:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177078.778650:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:0.0:1786177078.778655:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778661:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177078.778663:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177078.778665:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177078.778667:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856334464, offset 0
00000400:00000200:0.0:1786177078.778671:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177078.778674:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177078.778677:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177078.778680:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177078.778682:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177078.778685:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177078.778687:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177078.778690:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254784:254784:256:4294967295] 192.168.202.110@tcp LPNI seq info [254773:254773:8:4294967295]
00000400:00000200:0.0:1786177078.778694:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177078.778699:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177078.778704:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177078.778706:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177078.778710:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177078.778713:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786177078.778717:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177078.778719:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786177078.778723:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177078.778724:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177078.778727:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177078.778730:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177078.778733:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177078.778734:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177078.778736:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177078.778739:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778747:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177078.778751:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778757:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.778759:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177078.778762:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177078.778765:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177078.778768:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177078.778770:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301974547
00000100:00000001:0.0:1786177078.778773:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301978317
00000100:00000001:0.0:1786177078.778775:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301974547
00000100:00000001:0.0:1786177078.778777:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301978317
00000100:00000001:0.0:1786177078.778779:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177078.778781:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.778783:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177078.778786:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177078/real 1786177078]  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177078.778794:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177078.778795:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177078.778797:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177078.778799:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177078.778800:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177078.778802:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177078.778805:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177078.778813:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778819:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177078.778821:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778822:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177078.778824:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177078.778827:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778833:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177078.778837:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778843:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177078.778845:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778847:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177078.778848:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778849:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177078.778851:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177078.778853:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177078.778856:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177078.778858:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177078.778862:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177078.778869:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177078.778873:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778874:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177078.778877:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd180 x1872934856334080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778883:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177078.778885:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177078.778887:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177078.778889:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177078.778890:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177078.778892:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177078.778894:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177078.778895:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:0.0:1786177078.778899:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177078.778900:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177078.778901:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd180.
00000100:00000001:0.0:1786177078.778904:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177078.778919:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778922:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177078.778926:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc700 x1872934856334208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778932:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177078.778933:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177078.778935:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4800.
02000000:00000001:0.0:1786177078.778937:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177078.778939:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177078.778940:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177078.778942:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177078.778943:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:0.0:1786177078.778945:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177078.778947:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177078.778948:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc700.
00000100:00000001:0.0:1786177078.778950:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177078.778951:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778953:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177078.778956:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd500 x1872934856334336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778962:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177078.778963:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177078.778964:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786177078.778966:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177078.778967:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177078.778969:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177078.778971:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177078.778972:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:0.0:1786177078.778974:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177078.778976:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177078.778977:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd500.
00000100:00000001:0.0:1786177078.778978:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177078.778980:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.778981:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177078.778984:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856334464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177078.778990:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177078.778992:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177078.778993:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177078.778995:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177078.778996:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177078.778997:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177078.778999:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177078.779000:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:0.0:1786177078.779002:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177078.779003:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177078.779005:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:0.0:1786177078.779006:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177078.779007:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.779010:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177078.779013:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177078.779015:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177078.779017:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177078.779019:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177079.352621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177079.352627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177079.352631:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177079.352632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177079.352634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177079.352635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177079.352718:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177079.352724:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177079.352729:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177079.800606:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177079.800614:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177079.800617:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177079.800623:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177079.800627:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177079.800630:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177079.800633:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177079.800636:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177079.800638:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177080.376645:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177080.376652:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177080.376657:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177080.376659:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177080.376663:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177080.376664:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177080.376780:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177080.376786:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177080.376790:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177080.824637:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177080.824646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177080.824649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177080.824655:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177080.824659:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177080.824663:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177080.824666:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177080.824670:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177080.824672:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177081.400626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177081.400642:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177081.400644:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177081.400647:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177081.400651:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177081.400653:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177081.400654:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177081.400656:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177081.400659:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177081.400660:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177081.400661:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177081.400662:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177081.400663:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177081.400664:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177081.400665:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177081.400666:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177081.400668:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177081.400669:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177081.400736:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177081.400740:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177081.400743:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177081.848601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177081.848607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177081.848609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177081.848614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177081.848617:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177081.848620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177081.848622:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177081.848624:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177081.848627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177082.424616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177082.424621:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177082.424625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177082.424626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177082.424628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177082.424629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177082.424679:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177082.424682:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177082.424685:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177082.616594:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301978317, at 4301978368
00000100:00000040:1.0:1786177082.616594:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301978317, at 4301978368
00000100:00000010:0.0:1786177082.616608:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000010:1.0:1786177082.616611:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134300.
00000100:00000001:3.0:1786177082.872640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177082.872648:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177082.872650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177082.872656:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177082.872660:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177082.872664:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177082.872667:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177082.872670:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177082.872673:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177083.448615:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177083.448621:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177083.448626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177083.448627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177083.448629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177083.448632:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177083.448634:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177083.448637:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177083.448639:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177083.448642:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177083.448643:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177083.448645:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177083.448689:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177083.448693:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177083.448695:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177083.896573:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177083.896579:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177083.896581:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177083.896586:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177083.896590:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177083.896593:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.896595:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177083.896598:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177083.896602:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177083.896605:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177083.896611:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177083.896614:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177083.896617:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177083.896621:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177083.896627:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7307
00000400:00000200:0.0:1786177083.896635:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177083.896640:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177083.896643:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177083.896645:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177083.896648:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177083.896650:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177083.896651:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177083.896654:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177083.896656:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177083.896658:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177083.896661:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177083.896662:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177083.896664:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177083.896666:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177083.896667:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177083.896669:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177083.896671:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177083.896673:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177083.896675:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177083.896677:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177083.896679:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177083.896682:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177083.896684:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.896685:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177083.896687:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177083.896694:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418380.
00000020:00000040:0.0:1786177083.896696:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177083.896699:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177083.896700:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.896703:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177083.896704:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177083.896708:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177083.896714:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
00000100:00000001:0.0:1786177083.896719:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177083.896727:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177083.896734:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177083.896738:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177083.896749:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e6b901d7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177083.896769:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177083.896772:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177083.896774:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177083.896775:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177083.896776:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177083.896778:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177083.896780:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7307
00000400:00000200:0.0:1786177083.896783:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177083.896785:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177083.896787:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177083.896789:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177083.896791:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177083.896792:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177083.896795:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177083.896796:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177083.896797:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177083.896798:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177083.896800:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177083.896802:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:3.0:1786177083.896802:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1786177083.896806:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:3.0:1786177083.896807:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1786177083.896811:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177083.896811:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1786177083.896816:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177083.896820:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:3.0:1786177083.896821:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177083.896822:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177083.896828:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177083.896834:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177083.896839:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.896841:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1786177083.896842:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177083.896844:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177083.896848:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177083.896849:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177083.896853:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177083.896853:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177083.896870:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:3.0:1786177083.896873:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1786177083.896876:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00100000:3.0:1786177083.896879:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856334592:192.168.202.110@tcp:250:kworker.0
00000008:00000001:0.0:1786177083.896880:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.896884:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.896886:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:0.0:1786177083.896889:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b800.
00000020:00000040:0.0:1786177083.896892:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177083.896892:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177083.896896:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177083.896897:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177083.896901:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.896903:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177083.896903:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1786177083.896905:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177083.896908:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177083.896912:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
02000000:00000010:3.0:1786177083.896912:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:0.0:1786177083.896919:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177083.896919:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177083.896928:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177083.896933:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177083.896940:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177083.896943:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177083.896943:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000040:0.0:1786177083.896950:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000192ee9a0] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1786177083.896955:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177083.896957:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177083.896961:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177083.896963:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000200:3.0:1786177083.896964:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856334592, portal 25
00000100:00000001:0.0:1786177083.896965:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177083.896967:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177083.896971:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177083.896972:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177083.896977:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7307
00000100:00000001:3.0:1786177083.896977:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761655680 : -114547947895936 : ffff97d1ba418380)
00000400:00000200:0.0:1786177083.896983:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177083.896986:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177083.896989:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177083.896991:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177083.896993:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177083.896997:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177083.897001:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177083.897005:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177083.897009:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177083.897011:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177083.897015:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177083.897016:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:0.0:1786177083.897017:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177083.897021:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177083.897026:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177083.897027:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:0.0:1786177083.897030:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177083.897033:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1786177083.897034:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000040:0.0:1786177083.897035:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177083.897041:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177083.897043:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000200:3.0:1786177083.897044:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856334592, offset 0
00000100:00080000:0.0:1786177083.897045:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177083.897050:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177083.897051:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177083.897053:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177083.897058:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177083.897058:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177083.897060:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177083.897063:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177083.897066:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.897068:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177083.897072:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1786177083.897075:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418000.
00000400:00000200:3.0:1786177083.897080:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:0.0:1786177083.897081:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:3.0:1786177083.897087:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1786177083.897088:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177083.897091:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.897093:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177083.897095:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177083.897096:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177083.897100:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000010:0.0:1786177083.897102:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000400:00000200:3.0:1786177083.897106:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1786177083.897109:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177083.897110:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177083.897116:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254785:254785:256:4294967295] 192.168.202.110@tcp LPNI seq info [254774:254774:8:4294967295]
00000100:00100000:0.0:1786177083.897121:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177083.897124:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177083.897136:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177083.897137:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177083.897142:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177083.897147:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:0.0:1786177083.897153:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002e10e4f5] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:3.0:1786177083.897154:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177083.897165:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177083.897171:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:0.0:1786177083.897172:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177083.897176:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:3.1:1786177083.897177:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1786177083.897182:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000200:3.1:1786177083.897183:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:0.0:1786177083.897185:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177083.897188:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000100:3.0:1786177083.897192:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177083.897193:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:3.0:1786177083.897195:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00080000:0.0:1786177083.897198:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7307
00000400:00000200:3.0:1786177083.897202:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177083.897203:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177083.897209:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1786177083.897211:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:0.0:1786177083.897213:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1786177083.897217:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000100:00000040:0.0:1786177083.897218:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:3.0:1786177083.897222:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:0.0:1786177083.897223:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177083.897228:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177083.897229:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1786177083.897234:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:3.0:1786177083.897236:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177083.897238:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177083.897242:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177083.897246:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177083.897249:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1786177083.897251:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000040:3.0:1786177083.897254:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177083.897257:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177083.897265:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:3.0:1786177083.897265:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.897268:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:0.0:1786177083.897275:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177083.897275:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.897282:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177083.897282:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177083.897287:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301983437
00000100:00000040:0.0:1786177083.897289:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177083.897293:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000010:3.0:1786177083.897293:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37f60.
00000020:00000040:0.0:1786177083.897300:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:3.0:1786177083.897301:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:0.0:1786177083.897306:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177083.897308:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301983437
00000100:00000001:0.0:1786177083.897310:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177083.897313:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad373c0.
02000000:00000001:0.0:1786177083.897315:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177083.897317:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:0.0:1786177083.897322:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177083.897322:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177083.897326:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177083.897328:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.897330:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1786177083.897333:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00100000:3.0:1786177083.897337:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177083/real 1786177083]  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1786177083.897339:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177083.897346:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.897351:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177083.897358:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418700.
00000400:00000001:3.0:1786177083.897360:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:0.0:1786177083.897364:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000400:00000001:3.0:1786177083.897365:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177083.897369:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
02000000:00000001:0.0:1786177083.897370:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177083.897375:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177083.897375:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177083.897380:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177083.897381:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177083.897385:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177083.897386:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:0.0:1786177083.897388:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177083.897394:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786177083.897396:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4c00.
00000100:00000001:0.0:1786177083.897404:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177083.897407:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177083.897415:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177083.897416:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177083.897419:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.897423:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177083.897426:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177083.897434:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00100000:3.0:1786177083.897434:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177083.897440:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.897447:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177083.897451:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bf66f89d] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177083.897455:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.897485:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177083.897487:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.897489:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.897492:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:0.0:1786177083.897494:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177083.897494:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.897497:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177083.897500:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177083.897503:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00080000:3.0:1786177083.897504:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:0.0:1786177083.897507:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177083.897539:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177083.897541:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177083.897544:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177083.897553:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1786177083.897554:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856334592:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177083.897562:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.897565:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177083.897566:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:3.0:1786177083.897569:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418380 x1872934856334592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177083.897574:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7307
00000400:00000200:0.0:1786177083.897586:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177083.897587:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177083.897590:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177083.897593:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
00000400:00000200:0.0:1786177083.897595:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:3.0:1786177083.897597:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00080000:0.0:1786177083.897602:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000020:00000001:3.0:1786177083.897607:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000040:0.0:1786177083.897608:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000020:00000040:3.0:1786177083.897610:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:0.0:1786177083.897613:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000020:00000001:3.0:1786177083.897614:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177083.897618:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
00000100:00000040:0.0:1786177083.897620:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177083.897622:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177083.897625:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000001:0.0:1786177083.897628:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000010:3.0:1786177083.897629:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418380.
00000100:00000001:3.0:1786177083.897634:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000020:00000001:0.0:1786177083.897635:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177083.897636:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1786177083.897639:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:3.0:1786177083.897640:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1786177083.897646:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177083.897649:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000020:00000040:0.0:1786177083.897650:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:3.0:1786177083.897653:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177083.897656:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:3.0:1786177083.897657:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177083.897659:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177083.897662:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1786177083.897663:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177083.897667:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177083.897671:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177083.897677:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177083.897681:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1786177083.897682:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177083.897685:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.897687:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177083.897690:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1786177083.897693:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
02000000:00000001:3.0:1786177083.897695:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1786177083.897700:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1786177083.897701:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.897706:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177083.897710:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334720:192.168.202.110@tcp:38:kworker.0
02000000:00000001:0.0:1786177083.897711:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177083.897717:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177083.897718:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177083.897722:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177083.897723:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177083.897727:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.897728:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:3.0:1786177083.897730:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:0.0:1786177083.897732:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000010:3.0:1786177083.897734:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
00000008:00000001:0.0:1786177083.897738:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177083.897739:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.897744:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177083.897745:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177083.897752:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000010:0.0:1786177083.897753:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41bb80.
00000400:00000010:3.0:1786177083.897755:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000020:00000040:0.0:1786177083.897759:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000200:3.0:1786177083.897760:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856334720, portal 10
00000100:00000001:3.0:1786177083.897765:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1786177083.897766:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177083.897767:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761669120 : -114547947882496 : ffff97d1ba41b800)
02000000:00000001:0.0:1786177083.897770:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177083.897775:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:3.0:1786177083.897776:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177083.897780:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177083.897784:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.897785:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177083.897789:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177083.897792:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
02000000:00000010:0.0:1786177083.897794:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000200:3.0:1786177083.897795:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856334720, offset 0
00000400:00000200:3.0:1786177083.897801:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177083.897802:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177083.897806:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177083.897813:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:0.0:1786177083.897815:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177083.897816:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177083.897821:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177083.897825:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177083.897829:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177083.897835:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254786:254786:256:4294967295] 192.168.202.110@tcp LPNI seq info [254775:254775:8:4294967295]
00000400:00000200:3.0:1786177083.897845:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177083.897869:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177083.897871:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000200:3.0:1786177083.897876:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177083.897878:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1786177083.897880:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000040:0.0:1786177083.897887:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9983a05] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1786177083.897888:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177083.897892:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177083.897900:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177083.897905:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177083.897913:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177083.897915:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1786177083.897916:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177083.897921:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1786177083.897925:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7317)
00000400:00000200:3.0:1786177083.897925:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177083.897931:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177083.897934:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177083.897936:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177083.897939:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.897945:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177083.897948:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.897953:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.897954:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177083.897957:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177083.897960:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177083.897962:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177083.897963:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301979658
00000100:00000001:3.0:1786177083.897965:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301983437
00000100:00000001:3.0:1786177083.897967:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301979658
00000100:00000001:3.0:1786177083.897968:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301983437
00000100:00000001:3.0:1786177083.897970:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177083.897971:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.897972:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177083.897975:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177083/real 1786177083]  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177083.897979:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177083.897980:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177083.897981:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177083.897983:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177083.897984:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177083.897985:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177083.897987:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177083.897992:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898001:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177083.898002:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898003:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177083.898004:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177083.898007:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898011:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177083.898013:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898017:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177083.898019:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898020:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177083.898021:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898022:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177083.898023:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177083.898025:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177083.898027:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177083.898029:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177083.898031:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177083.898036:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177083.898038:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177083.898040:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898045:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177083.898046:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177083.898048:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177083.898049:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177083.898051:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177083.898052:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177083.898055:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177083.898056:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177083.898057:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177083.898058:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177083.898060:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177083.898062:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177083.898064:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177083.898066:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177083.898068:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177083.898070:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856334848, portal 10
00000100:00000001:3.0:1786177083.898071:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177083.898072:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761654784 : -114547947896832 : ffff97d1ba418000)
00000100:00000040:3.0:1786177083.898076:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898080:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177083.898081:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177083.898083:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177083.898084:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856334848, offset 0
00000400:00000200:3.0:1786177083.898087:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177083.898089:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177083.898091:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177083.898093:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177083.898095:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177083.898097:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177083.898098:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177083.898100:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254787:254787:256:4294967295] 192.168.202.110@tcp LPNI seq info [254776:254776:8:4294967295]
00000400:00000200:3.0:1786177083.898103:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177083.898107:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177083.898111:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177083.898113:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177083.898115:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177083.898117:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177083.898120:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177083.898121:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177083.898124:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177083.898125:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177083.898127:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177083.898129:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177083.898131:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177083.898132:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177083.898133:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177083.898136:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898141:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177083.898142:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898147:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.898148:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177083.898149:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177083.898151:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177083.898153:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177083.898154:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301979783
00000100:00000001:3.0:1786177083.898155:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301983437
00000100:00000001:3.0:1786177083.898157:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301979783
00000100:00000001:3.0:1786177083.898158:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301983437
00000100:00000001:3.0:1786177083.898160:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177083.898161:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.898162:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177083.898164:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177083/real 1786177083]  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177083.898170:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177083.898171:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177083.898172:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177083.898173:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177083.898174:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177083.898175:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177083.898177:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177083.898182:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898186:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177083.898187:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898188:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177083.898189:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177083.898192:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898196:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177083.898198:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898202:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177083.898203:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898204:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177083.898205:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898206:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177083.898207:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177083.898209:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177083.898210:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177083.898212:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177083.898215:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177083.898220:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177083.898222:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177083.898224:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898228:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177083.898229:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177083.898230:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177083.898231:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177083.898232:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177083.898234:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177083.898236:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177083.898237:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177083.898237:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177083.898238:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177083.898241:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177083.898243:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177083.898244:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177083.898246:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177083.898248:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177083.898250:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856334976, portal 4
00000100:00000001:3.0:1786177083.898251:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177083.898252:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761656576 : -114547947895040 : ffff97d1ba418700)
00000100:00000040:3.0:1786177083.898256:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898260:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177083.898261:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177083.898262:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177083.898264:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856334976, offset 0
00000400:00000200:3.0:1786177083.898267:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177083.898269:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177083.898271:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177083.898273:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177083.898274:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177083.898276:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177083.898278:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177083.898280:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254788:254788:256:4294967295] 192.168.202.110@tcp LPNI seq info [254777:254777:8:4294967295]
00000400:00000200:3.0:1786177083.898282:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177083.898286:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177083.898288:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177083.898290:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177083.898293:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177083.898294:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177083.898297:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177083.898298:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177083.898301:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177083.898302:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177083.898304:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177083.898306:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177083.898308:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177083.898309:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177083.898310:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177083.898312:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898317:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177083.898319:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898323:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.898324:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177083.898325:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177083.898327:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177083.898329:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177083.898331:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301979658
00000100:00000001:3.0:1786177083.898333:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301983437
00000100:00000001:3.0:1786177083.898334:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301979658
00000100:00000001:3.0:1786177083.898336:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301983437
00000100:00000001:3.0:1786177083.898338:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177083.898340:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.898342:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177083.898345:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177083/real 1786177083]  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177083.898353:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177083.898355:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177083.898356:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177083.898358:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177083.898360:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177083.898363:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177083.898366:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177083.898376:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898384:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177083.898385:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898387:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177083.898388:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177083.898392:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898397:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177083.898399:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898404:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177083.898406:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898407:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177083.898408:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898409:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177083.898410:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177083.898412:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177083.898414:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177083.898415:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177083.898418:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177083.898422:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856334976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177083.898424:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898426:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177083.898428:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b800 x1872934856334720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898432:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177083.898433:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177083.898435:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177083.898437:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177083.898437:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177083.898438:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177083.898440:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177083.898441:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:3.0:1786177083.898443:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177083.898444:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177083.898445:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b800.
00000100:00000001:3.0:1786177083.898447:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177083.898448:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898449:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177083.898451:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418000 x1872934856334848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898460:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177083.898461:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177083.898462:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177083.898463:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177083.898464:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177083.898465:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177083.898466:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177083.898467:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:3.0:1786177083.898468:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177083.898469:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177083.898470:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418000.
00000100:00000001:3.0:1786177083.898471:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177083.898472:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898473:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177083.898474:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418700 x1872934856334976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898478:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177083.898479:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177083.898480:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177083.898481:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177083.898482:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177083.898482:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177083.898483:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177083.898484:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:3.0:1786177083.898485:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177083.898486:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177083.898487:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418700.
00000100:00000001:3.0:1786177083.898488:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177083.898489:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898490:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898492:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177083.898493:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.898494:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177083.898495:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177083.898496:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177083.898499:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898503:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177083.898504:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177083.898506:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177083.898507:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177083.898523:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177083.898526:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856335104:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177083.898528:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177083.898530:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177083.898531:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177083.898532:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177083.898534:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177083.898536:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177083.898538:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177083.898540:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177083.898542:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177083.898544:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856335104, portal 4
00000100:00000001:3.0:1786177083.898545:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177083.898546:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761670016 : -114547947881600 : ffff97d1ba41bb80)
00000100:00000040:3.0:1786177083.898551:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898555:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177083.898556:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177083.898558:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177083.898560:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856335104, offset 0
00000400:00000200:3.0:1786177083.898562:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177083.898565:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177083.898567:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177083.898569:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177083.898571:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177083.898572:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177083.898574:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177083.898576:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254789:254789:256:4294967295] 192.168.202.110@tcp LPNI seq info [254778:254778:8:4294967295]
00000400:00000200:3.0:1786177083.898579:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177083.898583:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177083.898586:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177083.898588:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177083.898591:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177083.898593:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177083.898596:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177083.898598:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177083.898600:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177083.898602:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177083.898603:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177083.898606:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177083.898608:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177083.898609:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177083.898611:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177083.898613:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898618:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177083.898620:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898624:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.898625:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177083.898627:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177083.898629:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177083.898631:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177083.898633:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301979667
00000100:00000001:3.0:1786177083.898634:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301983437
00000100:00000001:3.0:1786177083.898636:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301979667
00000100:00000001:3.0:1786177083.898637:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301983437
00000100:00000001:3.0:1786177083.898639:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177083.898640:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.898642:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177083.898644:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177083/real 1786177083]  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177083.898649:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177083.898650:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177083.898651:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177083.898652:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177083.898653:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177083.898654:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177083.898656:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177083.898662:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898665:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177083.898666:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898667:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177083.898668:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177083.898670:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898674:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177083.898678:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898685:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177083.898687:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898689:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177083.898690:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898692:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177083.898693:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177083.898695:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177083.898699:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177083.898701:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177083.898706:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177083.898715:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856335104:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177083.898719:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898721:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177083.898724:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41bb80 x1872934856335104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177083.898731:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177083.898732:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177083.898734:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177083.898735:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177083.898736:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177083.898737:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177083.898738:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177083.898740:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:3.0:1786177083.898741:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177083.898742:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177083.898743:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41bb80.
00000100:00000001:3.0:1786177083.898745:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177083.898746:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898747:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177083.898749:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177083.898750:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177083.898752:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177083.898753:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177084.472622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177084.472629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177084.472635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177084.472637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177084.472641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177084.472642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177084.472707:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177084.472712:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177084.472716:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177084.920629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177084.920637:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177084.920639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177084.920645:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177084.920649:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177084.920652:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177084.920656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177084.920659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177084.920663:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177085.496586:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177085.496592:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177085.496598:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177085.496600:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177085.496603:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177085.496605:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177085.496646:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177085.496650:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177085.496653:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177085.944641:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177085.944649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177085.944651:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177085.944657:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177085.944661:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177085.944666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177085.944669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177085.944672:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177085.944675:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177086.520633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177086.520638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177086.520643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177086.520644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177086.520646:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177086.520647:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177086.520711:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177086.520716:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177086.520721:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177086.968590:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177086.968600:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177086.968602:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177086.968610:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177086.968614:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177086.968619:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177086.968623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177086.968627:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177086.968630:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177087.544604:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177087.544612:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177087.544617:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177087.544618:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177087.544620:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177087.544621:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177087.544677:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177087.544682:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177087.544685:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177087.736672:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301983437, at 4301983488
00000100:00000040:0.0:1786177087.736681:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301983437, at 4301983488
00000100:00000010:1.0:1786177087.736690:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad373c0.
00000100:00000010:0.0:1786177087.736700:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37f60.
00000100:00000001:3.0:1786177087.992678:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177087.992687:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177087.992689:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177087.992695:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177087.992699:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177087.992702:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177087.992705:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177087.992708:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177087.992711:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177088.568648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177088.568660:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177088.568668:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177088.568669:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177088.568673:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177088.568675:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177088.568740:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177088.568746:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177088.568750:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177089.016588:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177089.016597:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177089.016599:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177089.016605:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177089.016609:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177089.016613:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177089.016615:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177089.016616:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177089.016620:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.016626:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177089.016627:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177089.016632:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177089.016637:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177089.016642:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177089.016652:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7312
00000400:00000200:0.0:1786177089.016663:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177089.016672:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177089.016676:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177089.016680:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177089.016684:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177089.016687:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177089.016691:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177089.016695:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177089.016698:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177089.016702:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177089.016707:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177089.016710:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177089.016713:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177089.016717:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177089.016719:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177089.016721:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177089.016723:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177089.016727:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177089.016731:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177089.016734:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177089.016738:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177089.016742:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177089.016746:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177089.016748:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177089.016751:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177089.016764:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419c00.
00000020:00000040:0.0:1786177089.016769:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177089.016774:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177089.016777:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177089.016781:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177089.016785:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177089.016790:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177089.016801:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6000.
00000100:00000001:0.0:1786177089.016822:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177089.016835:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177089.016848:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177089.016853:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177089.016866:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cf890c1c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177089.016893:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177089.016897:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177089.016902:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177089.016904:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177089.016907:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177089.016911:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177089.016914:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1786177089.016915:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7312
00000100:00000001:3.0:1786177089.016919:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000400:00000200:0.0:1786177089.016924:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177089.016924:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177089.016932:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177089.016933:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177089.016940:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177089.016947:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1786177089.016948:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177089.016949:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177089.016953:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1786177089.016955:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1786177089.016959:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177089.016960:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177089.016965:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177089.016965:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177089.016972:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00100000:3.0:1786177089.016972:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856335232:192.168.202.110@tcp:250:kworker.0
00000020:00000040:0.0:1786177089.016975:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177089.016980:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177089.016980:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:0.0:1786177089.016985:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
02000000:00000001:3.0:1786177089.016988:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:0.0:1786177089.016989:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177089.016993:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:3.0:1786177089.016994:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177089.017000:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177089.017001:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1786177089.017004:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177089.017008:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000010:3.0:1786177089.017010:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:0.0:1786177089.017011:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177089.017017:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177089.017018:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177089.017024:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177089.017027:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177089.017029:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786177089.017031:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1786177089.017036:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
02000000:00000001:0.0:1786177089.017040:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177089.017044:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177089.017045:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177089.017048:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177089.017055:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000200:3.0:1786177089.017055:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856335232, portal 25
00000100:00000001:3.0:1786177089.017061:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000008:00000001:0.0:1786177089.017063:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.017064:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761661952 : -114547947889664 : ffff97d1ba419c00)
00000100:00000001:0.0:1786177089.017069:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177089.017073:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1786177089.017076:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a300.
00000020:00000040:0.0:1786177089.017082:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177089.017089:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177089.017090:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1786177089.017092:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177089.017097:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:3.0:1786177089.017098:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786177089.017102:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1786177089.017105:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
02000000:00000001:0.0:1786177089.017106:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177089.017112:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856335232, offset 0
02000000:00000010:0.0:1786177089.017114:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4000.
00000100:00000001:0.0:1786177089.017122:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177089.017127:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00100000:0.0:1786177089.017132:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177089.017140:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177089.017147:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177089.017151:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1786177089.017153:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177089.017159:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1786177089.017161:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b0d3569] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177089.017165:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177089.017171:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177089.017178:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177089.017184:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254790:254790:256:4294967295] 192.168.202.110@tcp LPNI seq info [254779:254779:8:4294967295]
00000100:00000001:0.0:1786177089.017185:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177089.017192:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1786177089.017193:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177089.017200:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177089.017202:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177089.017206:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1786177089.017207:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177089.017214:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:3.0:1786177089.017218:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177089.017221:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7312
00000800:00000010:3.0:1786177089.017225:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000400:00000200:0.0:1786177089.017230:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177089.017237:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:3.0:1786177089.017238:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1786177089.017241:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1786177089.017245:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:0.0:1786177089.017247:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177089.017253:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177089.017257:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:3.1:1786177089.017262:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177089.017263:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177089.017267:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177089.017268:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177089.017270:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177089.017272:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177089.017276:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177089.017281:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:3.1:1786177089.017281:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:0.0:1786177089.017291:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000100:3.0:1786177089.017291:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177089.017296:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00000001:0.0:1786177089.017297:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177089.017302:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177089.017304:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1786177089.017306:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177089.017311:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:3.0:1786177089.017312:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:0.0:1786177089.017319:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1786177089.017321:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000100:00000001:0.0:1786177089.017324:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177089.017325:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
02000000:00000001:0.0:1786177089.017330:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177089.017334:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:0.0:1786177089.017335:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177089.017340:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177089.017344:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000200:3.0:1786177089.017344:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1786177089.017348:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177089.017354:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177089.017359:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177089.017366:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a680.
00000100:00000001:3.0:1786177089.017369:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177089.017375:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177089.017381:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000001:3.0:1786177089.017385:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177089.017387:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177089.017388:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:0.0:1786177089.017391:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177089.017396:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177089.017397:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177089.017398:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177089.017403:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177089.017404:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177089.017409:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301988557
02000000:00000010:0.0:1786177089.017411:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5800.
00000100:00000001:0.0:1786177089.017416:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177089.017416:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37c60.
00000100:00000001:3.0:1786177089.017426:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00100000:0.0:1786177089.017427:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177089.017434:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301988557
00000100:00000010:3.0:1786177089.017439:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37120.
00000100:00000001:3.0:1786177089.017445:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786177089.017451:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177089.017452:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177089.017458:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.017458:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.017464:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:0.0:1786177089.017468:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000911ee1e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177089.017471:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177089/real 1786177089]  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177089.017502:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1786177089.017503:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177089.017505:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177089.017528:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177089.017531:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177089.017532:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177089.017535:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177089.017538:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177089.017540:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177089.017550:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177089.017555:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:3.0:1786177089.017555:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177089.017560:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177089.017567:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177089.017568:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177089.017570:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177089.017574:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7312
00000100:00000001:3.0:1786177089.017575:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177089.017578:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1786177089.017585:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1786177089.017585:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177089.017594:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177089.017596:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177089.017600:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177089.017604:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177089.017610:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177089.017615:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177089.017617:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177089.017620:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177089.017623:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177089.017624:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177089.017628:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177089.017630:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177089.017631:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177089.017634:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:0.0:1786177089.017636:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00080000:3.0:1786177089.017638:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000020:00000040:0.0:1786177089.017639:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177089.017644:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00080000:3.0:1786177089.017644:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177089.017648:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000020:00000040:0.0:1786177089.017649:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000040:3.0:1786177089.017653:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177089.017655:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177089.017662:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00100000:3.0:1786177089.017668:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856335232:192.168.202.110@tcp:250:kworker.0
00000100:00000040:0.0:1786177089.017671:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177089.017674:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177089.017676:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177089.017677:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177089.017683:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:3.0:1786177089.017684:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419c00 x1872934856335232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177089.017690:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177089.017693:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177089.017696:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000020:00000040:0.0:1786177089.017697:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
02000000:00000010:3.0:1786177089.017700:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
00000100:00080000:0.0:1786177089.017704:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1786177089.017705:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177089.017707:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000001:0.0:1786177089.017711:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177089.017712:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177089.017716:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000001:0.0:1786177089.017718:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:3.0:1786177089.017720:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786177089.017723:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177089.017725:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177089.017728:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
02000000:00000001:0.0:1786177089.017730:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177089.017731:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419c00.
00000100:00000001:3.0:1786177089.017736:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177089.017737:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177089.017738:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.017742:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:0.0:1786177089.017743:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177089.017748:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.017750:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177089.017751:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.017754:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.017756:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000010:0.0:1786177089.017758:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419500.
00000100:00000001:3.0:1786177089.017758:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177089.017761:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1786177089.017765:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000040:3.0:1786177089.017766:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177089.017771:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177089.017775:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177089.017779:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177089.017781:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1786177089.017783:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177089.017784:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177089.017789:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177089.017789:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177089.017792:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177089.017795:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177089.017796:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00100000:3.0:1786177089.017797:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856335360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177089.017803:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1786177089.017804:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177089.017805:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177089.017819:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177089.017823:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177089.017826:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00100000:0.0:1786177089.017830:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1786177089.017830:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177089.017837:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177089.017841:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177089.017846:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177089.017848:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:3.0:1786177089.017850:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856335360, portal 10
00000100:00000001:0.0:1786177089.017853:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.017854:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177089.017856:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761663744 : -114547947887872 : ffff97d1ba41a300)
00000100:00000040:0.0:1786177089.017861:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b6718a9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177089.017865:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.017893:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1786177089.017897:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177089.017897:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177089.017901:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00080000:0.0:1786177089.017903:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:3.0:1786177089.017904:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856335360, offset 0
00000400:00000200:3.0:1786177089.017912:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177089.017913:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177089.017916:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1786177089.017917:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177089.017919:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1786177089.017922:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1786177089.017925:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177089.017926:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177089.017933:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:0.0:1786177089.017934:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7312
00000400:00000200:3.0:1786177089.017937:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177089.017941:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177089.017945:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177089.017948:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254791:254791:256:4294967295] 192.168.202.110@tcp LPNI seq info [254780:254780:8:4294967295]
00000400:00000200:0.0:1786177089.017956:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1786177089.017956:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177089.017962:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1786177089.017966:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1786177089.017971:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:3.0:1786177089.017975:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177089.017977:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:3.0:1786177089.017980:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000040:0.0:1786177089.017983:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177089.017988:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.0:1786177089.017991:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:0.0:1786177089.017993:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177089.017995:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177089.017999:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:3.0:1786177089.017999:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000020:00000040:0.0:1786177089.018004:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177089.018009:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000800:00000200:3.1:1786177089.018010:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:0.0:1786177089.018013:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000800:00000200:3.1:1786177089.018017:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:0.0:1786177089.018021:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177089.018027:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:3.0:1786177089.018028:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:0.0:1786177089.018033:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000010:3.0:1786177089.018033:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177089.018038:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1786177089.018039:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177089.018043:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000400:00000200:3.0:1786177089.018046:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:0.0:1786177089.018051:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1786177089.018052:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177089.018056:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:0.0:1786177089.018057:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.018059:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:0.0:1786177089.018062:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1786177089.018066:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177089.018067:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177089.018074:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177089.018080:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177089.018082:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000008:00100000:0.0:1786177089.018084:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000040:3.0:1786177089.018088:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1786177089.018089:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177089.018094:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177089.018100:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41aa00.
00000100:00000001:3.0:1786177089.018104:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177089.018105:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177089.018108:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:0.0:1786177089.018111:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177089.018115:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.018116:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177089.018119:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:3.0:1786177089.018123:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
02000000:00000001:0.0:1786177089.018124:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177089.018127:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177089.018132:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1786177089.018135:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7800.
00000100:00000001:3.0:1786177089.018135:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301984778
00000100:00000001:3.0:1786177089.018138:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301988557
00000100:00000001:0.0:1786177089.018142:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.018142:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301984778
00000100:00000001:3.0:1786177089.018149:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301988557
00000100:00000001:3.0:1786177089.018151:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:0.0:1786177089.018154:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177089.018155:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.018159:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177089.018165:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177089/real 1786177089]  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177089.018174:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177089.018180:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177089.018180:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177089.018183:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177089.018186:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00000040:0.0:1786177089.018190:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da0255a4] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177089.018190:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177089.018193:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177089.018197:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177089.018203:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177089.018213:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177089.018217:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177089.018221:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7322)
00000100:00000001:3.0:1786177089.018223:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177089.018226:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018227:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177089.018229:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177089.018233:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018240:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177089.018244:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018252:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177089.018255:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018257:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177089.018258:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018260:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177089.018263:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177089.018266:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177089.018270:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177089.018273:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177089.018278:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177089.018284:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856335360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177089.018287:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177089.018289:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018294:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177089.018295:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177089.018297:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177089.018298:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177089.018299:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177089.018301:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856335488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177089.018304:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177089.018305:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177089.018306:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177089.018308:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177089.018311:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177089.018312:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177089.018315:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177089.018317:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177089.018319:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177089.018321:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856335488, portal 10
00000100:00000001:3.0:1786177089.018323:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177089.018324:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761664640 : -114547947886976 : ffff97d1ba41a680)
00000100:00000040:3.0:1786177089.018328:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018334:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177089.018337:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177089.018339:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177089.018342:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856335488, offset 0
00000400:00000200:3.0:1786177089.018346:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177089.018350:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177089.018354:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177089.018357:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177089.018360:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177089.018363:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177089.018366:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177089.018370:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254792:254792:256:4294967295] 192.168.202.110@tcp LPNI seq info [254781:254781:8:4294967295]
00000400:00000200:3.0:1786177089.018374:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177089.018382:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177089.018386:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177089.018389:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177089.018392:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177089.018394:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786177089.018398:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177089.018399:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1786177089.018402:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177089.018404:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177089.018406:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177089.018408:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177089.018411:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177089.018412:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177089.018414:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177089.018417:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018423:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177089.018425:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018430:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.018431:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177089.018433:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177089.018435:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177089.018438:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177089.018440:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301984903
00000100:00000001:3.0:1786177089.018442:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301988557
00000100:00000001:3.0:1786177089.018443:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301984903
00000100:00000001:3.0:1786177089.018445:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301988557
00000100:00000001:3.0:1786177089.018446:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177089.018447:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.018449:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177089.018451:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177089/real 1786177089]  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177089.018456:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177089.018457:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177089.018458:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177089.018460:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177089.018461:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177089.018463:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177089.018465:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177089.018470:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018474:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177089.018475:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018476:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177089.018477:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177089.018479:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018483:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177089.018486:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018491:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177089.018493:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018494:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177089.018495:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018496:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177089.018497:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177089.018499:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177089.018500:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177089.018502:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177089.018504:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177089.018525:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856335488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177089.018528:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018530:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177089.018532:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a300 x1872934856335360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018537:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177089.018538:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177089.018539:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177089.018541:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177089.018542:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177089.018543:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177089.018545:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177089.018546:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4000.
02000000:00000001:3.0:1786177089.018548:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177089.018549:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177089.018550:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a300.
00000100:00000001:3.0:1786177089.018552:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177089.018553:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018554:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177089.018556:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a680 x1872934856335488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018561:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177089.018562:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177089.018563:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177089.018564:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177089.018565:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177089.018566:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177089.018567:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177089.018568:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5800.
02000000:00000001:3.0:1786177089.018569:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177089.018570:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177089.018570:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a680.
00000100:00000001:3.0:1786177089.018572:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177089.018572:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018573:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018576:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177089.018577:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.018578:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177089.018579:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177089.018580:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177089.018582:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018586:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177089.018587:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177089.018588:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177089.018589:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177089.018591:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177089.018592:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856335616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177089.018595:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177089.018596:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177089.018597:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177089.018599:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177089.018601:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177089.018602:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177089.018605:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177089.018606:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177089.018609:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177089.018611:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856335616, portal 4
00000100:00000001:3.0:1786177089.018612:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177089.018613:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761660160 : -114547947891456 : ffff97d1ba419500)
00000100:00000040:3.0:1786177089.018618:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018622:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177089.018624:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177089.018625:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177089.018627:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856335616, offset 0
00000400:00000200:3.0:1786177089.018629:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177089.018631:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177089.018634:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177089.018635:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177089.018637:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177089.018639:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177089.018641:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177089.018643:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254793:254793:256:4294967295] 192.168.202.110@tcp LPNI seq info [254782:254782:8:4294967295]
00000400:00000200:3.0:1786177089.018646:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177089.018650:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177089.018653:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177089.018654:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177089.018657:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177089.018659:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786177089.018662:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177089.018664:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1786177089.018667:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177089.018668:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177089.018670:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177089.018673:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177089.018675:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177089.018676:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177089.018677:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177089.018680:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018685:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177089.018687:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018691:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.018693:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177089.018694:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177089.018697:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177089.018699:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177089.018700:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301984779
00000100:00000001:3.0:1786177089.018702:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301988557
00000100:00000001:3.0:1786177089.018704:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301984779
00000100:00000001:3.0:1786177089.018705:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301988557
00000100:00000001:3.0:1786177089.018707:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177089.018708:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.018709:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177089.018711:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177089/real 1786177089]  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177089.018716:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177089.018717:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177089.018718:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177089.018719:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177089.018720:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177089.018721:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177089.018723:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177089.018733:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018737:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177089.018738:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018739:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177089.018740:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177089.018742:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018746:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177089.018748:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018752:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177089.018754:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018755:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177089.018755:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018757:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177089.018758:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177089.018759:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177089.018761:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177089.018763:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177089.018765:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177089.018771:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856335616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177089.018773:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177089.018775:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018783:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177089.018784:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177089.018785:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177089.018787:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177089.018788:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177089.018790:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856335744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177089.018791:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177089.018792:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177089.018793:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177089.018794:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177089.018796:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177089.018797:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177089.018798:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177089.018800:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177089.018802:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177089.018803:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856335744, portal 4
00000100:00000001:3.0:1786177089.018805:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177089.018820:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761665536 : -114547947886080 : ffff97d1ba41aa00)
00000100:00000040:3.0:1786177089.018826:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018831:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177089.018832:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177089.018834:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177089.018836:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856335744, offset 0
00000400:00000200:3.0:1786177089.018839:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177089.018842:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177089.018845:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177089.018847:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177089.018848:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177089.018850:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177089.018852:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177089.018854:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254794:254794:256:4294967295] 192.168.202.110@tcp LPNI seq info [254783:254783:8:4294967295]
00000400:00000200:3.0:1786177089.018857:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177089.018862:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177089.018865:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177089.018867:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177089.018870:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177089.018872:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786177089.018875:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177089.018876:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1786177089.018879:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177089.018880:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177089.018882:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177089.018885:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177089.018887:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177089.018888:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177089.018890:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177089.018892:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018897:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177089.018899:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018904:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.018905:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177089.018907:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177089.018909:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177089.018911:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177089.018912:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301984787
00000100:00000001:3.0:1786177089.018914:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301988557
00000100:00000001:3.0:1786177089.018916:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301984787
00000100:00000001:3.0:1786177089.018917:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301988557
00000100:00000001:3.0:1786177089.018919:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177089.018920:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.018922:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177089.018924:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177089/real 1786177089]  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177089.018930:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177089.018931:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177089.018932:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177089.018933:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177089.018934:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177089.018935:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177089.018937:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177089.018943:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018947:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177089.018948:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018949:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177089.018950:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177089.018952:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018956:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177089.018958:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018962:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177089.018963:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018964:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177089.018965:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018966:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177089.018967:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177089.018969:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177089.018971:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177089.018973:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177089.018975:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177089.018983:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856335744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177089.018987:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.018989:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177089.018992:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419500 x1872934856335616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.018998:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177089.019000:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177089.019002:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177089.019005:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177089.019007:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177089.019008:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177089.019010:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177089.019012:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:3.0:1786177089.019015:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177089.019016:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177089.019018:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419500.
00000100:00000001:3.0:1786177089.019020:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177089.019022:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.019024:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177089.019028:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41aa00 x1872934856335744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177114 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177089.019034:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177089.019035:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177089.019037:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177089.019038:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177089.019039:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177089.019040:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177089.019041:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177089.019043:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7800.
02000000:00000001:3.0:1786177089.019045:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177089.019045:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177089.019046:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41aa00.
00000100:00000001:3.0:1786177089.019048:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177089.019049:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.019050:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177089.019052:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177089.019053:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177089.019055:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177089.019056:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177089.592646:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177089.592657:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177089.592665:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177089.592666:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177089.592670:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177089.592672:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177089.592736:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177089.592740:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177089.592744:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177090.040663:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177090.040675:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177090.040678:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177090.040686:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177090.040691:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177090.040696:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177090.040700:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177090.040704:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177090.040708:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177090.616584:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177090.616593:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177090.616595:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177090.616598:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177090.616603:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177090.616606:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177090.616607:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177090.616609:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177090.616612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177090.616614:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177090.616615:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177090.616616:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177090.616617:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177090.616618:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177090.616620:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177090.616621:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177090.616623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177090.616624:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177090.616681:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177090.616685:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177090.616688:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177091.064566:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177091.064572:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177091.064574:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177091.064579:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177091.064582:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177091.064585:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177091.064587:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177091.064589:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177091.064591:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177091.640591:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177091.640599:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177091.640604:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177091.640605:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177091.640608:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177091.640609:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177091.640669:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177091.640673:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177091.640676:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177092.088691:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177092.088704:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177092.088707:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177092.088715:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177092.088719:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177092.088722:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177092.088725:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177092.088728:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177092.088730:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177092.664664:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177092.664671:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177092.664676:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177092.664677:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177092.664679:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177092.664682:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177092.664684:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177092.664688:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177092.664691:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177092.664693:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177092.664695:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177092.664697:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177092.664761:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177092.664765:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177092.664768:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177092.856640:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301988557, at 4301988608
00000100:00000040:1.0:1786177092.856640:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301988557, at 4301988608
00000100:00000010:1.0:1786177092.856659:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37120.
00000100:00000010:0.0:1786177092.856660:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37c60.
00000100:00000001:3.0:1786177093.112663:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177093.112676:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177093.112679:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177093.112687:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177093.112692:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177093.112697:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177093.112701:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177093.112705:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177093.112709:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177093.688646:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177093.688653:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177093.688660:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177093.688661:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177093.688664:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177093.688666:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177093.688775:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177093.688781:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177093.688786:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177094.136621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:0.0:1786177094.136623:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177094.136630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786177094.136632:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177094.136634:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.136635:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177094.136640:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1786177094.136641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177094.136645:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177094.136647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1786177094.136652:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7317
00000100:00000001:3.0:1786177094.136653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.136659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000400:00000200:0.0:1786177094.136663:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177094.136663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.136670:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177094.136674:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177094.136678:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177094.136680:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.136684:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177094.136686:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.136688:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177094.136691:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177094.136693:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177094.136695:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177094.136698:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177094.136699:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177094.136701:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177094.136704:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.136705:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177094.136706:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.136708:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177094.136710:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177094.136712:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177094.136714:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177094.136716:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177094.136719:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177094.136721:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.136722:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177094.136723:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177094.136731:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41ad80.
00000020:00000040:0.0:1786177094.136734:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177094.136737:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177094.136738:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.136740:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177094.136743:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177094.136746:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177094.136754:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000100:00000001:0.0:1786177094.136773:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177094.136782:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177094.136795:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177094.136799:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177094.136808:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034e2c676] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177094.136823:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177094.136825:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177094.136828:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177094.136829:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177094.136830:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177094.136832:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177094.136834:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7317
00000400:00000200:0.0:1786177094.136837:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177094.136839:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177094.136841:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177094.136843:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.136845:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177094.136846:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.136848:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177094.136849:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177094.136850:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177094.136851:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177094.136853:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177094.136855:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177094.136857:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177094.136859:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.136860:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177094.136862:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177094.136863:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177094.136864:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177094.136867:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1786177094.136869:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:3.0:1786177094.136871:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1786177094.136873:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177094.136876:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177094.136879:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1786177094.136880:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177094.136884:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177094.136888:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.136892:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177094.136895:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000008:00100000:0.0:1786177094.136896:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1786177094.136898:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1786177094.136903:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177094.136904:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1786177094.136907:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177094.136909:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:0.0:1786177094.136912:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b480.
02000000:00000001:3.0:1786177094.136912:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177094.136918:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00100000:3.0:1786177094.136919:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856335872:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1786177094.136921:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177094.136924:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.136926:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1786177094.136927:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177094.136931:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177094.136931:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177094.136935:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177094.136936:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177094.136937:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177094.136944:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000010:0.0:1786177094.136945:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7800.
02000000:00000001:3.0:1786177094.136948:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.136952:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177094.136956:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177094.136962:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00100000:0.0:1786177094.136964:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1786177094.136968:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177094.136977:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856335872, portal 25
00000100:00000001:0.0:1786177094.136978:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177094.136982:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177094.136983:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.136984:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761666432 : -114547947885184 : ffff97d1ba41ad80)
00000100:00000040:0.0:1786177094.136990:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001215e9e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177094.136994:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177094.137010:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.137011:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:0.0:1786177094.137013:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1786177094.137016:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177094.137019:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:3.0:1786177094.137020:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000001:0.0:1786177094.137022:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177094.137025:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1786177094.137025:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856335872, offset 0
00000100:00000001:0.0:1786177094.137027:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177094.137032:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7317
00000400:00000200:3.0:1786177094.137035:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177094.137039:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177094.137043:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1786177094.137046:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177094.137047:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177094.137052:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137054:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177094.137057:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:0.0:1786177094.137059:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137063:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177094.137064:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:0.0:1786177094.137065:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177094.137068:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177094.137069:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:3.0:1786177094.137071:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:0.0:1786177094.137073:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177094.137076:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177094.137077:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:0.0:1786177094.137078:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177094.137082:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:0.0:1786177094.137085:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137089:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177094.137090:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254795:254795:256:4294967295] 192.168.202.110@tcp LPNI seq info [254784:254784:8:4294967295]
00000100:00000040:0.0:1786177094.137093:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137098:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177094.137102:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1786177094.137103:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786177094.137107:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137109:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177094.137111:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177094.137114:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177094.137117:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:0.0:1786177094.137118:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.137122:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177094.137124:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000800:00000200:3.0:1786177094.137129:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000008:00000001:0.0:1786177094.137130:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.137134:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1786177094.137137:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000010:0.0:1786177094.137138:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b100.
00000020:00000040:0.0:1786177094.137142:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177094.137146:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:3.0:1786177094.137147:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:0.0:1786177094.137148:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.137151:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177094.137153:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:3.0:1786177094.137155:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
02000000:00000001:0.0:1786177094.137156:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177094.137160:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000001:0.0:1786177094.137164:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177094.137166:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:0.0:1786177094.137172:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1786177094.137174:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1786177094.137186:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137189:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000010:3.0:1786177094.137191:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00000001:0.0:1786177094.137194:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177094.137201:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1786177094.137202:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000fa632fb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177094.137213:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1786177094.137220:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177094.137221:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000100:00080000:0.0:1786177094.137223:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1786177094.137225:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00080000:0.0:1786177094.137230:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177094.137232:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1786177094.137233:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177094.137236:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1786177094.137240:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177094.137241:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177094.137246:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7317
00000400:00000200:0.0:1786177094.137254:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177094.137259:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177094.137260:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1786177094.137263:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177094.137267:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177094.137269:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137274:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177094.137278:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137284:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177094.137284:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177094.137289:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177094.137289:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:0.0:1786177094.137292:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177094.137295:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177094.137297:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:3.0:1786177094.137297:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177094.137303:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:3.0:1786177094.137307:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:0.0:1786177094.137308:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177094.137313:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177094.137313:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301993677
00000100:00000001:0.0:1786177094.137318:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177094.137319:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137321:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000010:3.0:1786177094.137321:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad379c0.
00000020:00000040:0.0:1786177094.137326:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177094.137328:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177094.137329:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786177094.137330:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177094.137334:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177094.137336:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301993677
02000000:00000001:0.0:1786177094.137337:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177094.137342:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177094.137343:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad374e0.
00000100:00000001:0.0:1786177094.137344:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177094.137349:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177094.137349:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177094.137354:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00000001:0.0:1786177094.137355:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.137359:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.137360:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177094.137364:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419180.
00000100:00000001:3.0:1786177094.137364:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000020:00000040:0.0:1786177094.137368:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00100000:3.0:1786177094.137369:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177094/real 1786177094]  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177094.137372:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177094.137375:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.137379:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177094.137381:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177094.137385:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177094.137385:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000010:0.0:1786177094.137388:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5800.
00000400:00000001:3.0:1786177094.137389:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177094.137393:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00000001:0.0:1786177094.137394:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177094.137397:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177094.137400:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00100000:0.0:1786177094.137401:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177094.137402:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177094.137409:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177094.137413:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177094.137417:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177094.137424:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005a89d8ab] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177094.137426:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177094.137443:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177094.137448:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177094.137455:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177094.137458:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177094.137461:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177094.137461:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177094.137465:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177094.137465:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177094.137469:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7317
00000100:00000001:3.0:1786177094.137469:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177094.137471:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1786177094.137475:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1786177094.137479:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177094.137480:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177094.137484:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177094.137490:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137494:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177094.137496:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177094.137498:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137501:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177094.137504:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000040:3.0:1786177094.137504:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177094.137507:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:3.0:1786177094.137530:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177094.137532:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.137533:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000001:0.0:1786177094.137535:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177094.137536:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1786177094.137539:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:3.0:1786177094.137540:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000001:0.0:1786177094.137543:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:3.0:1786177094.137544:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:0.0:1786177094.137548:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00080000:3.0:1786177094.137549:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:0.0:1786177094.137554:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00080000:3.0:1786177094.137557:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177094.137558:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177094.137562:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177094.137563:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137567:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:3.0:1786177094.137570:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177094.137571:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177094.137576:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137582:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177094.137586:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177094.137591:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1786177094.137591:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856335872:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1786177094.137596:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.137599:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177094.137600:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:0.0:1786177094.137602:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177094.137604:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000008:00000001:0.0:1786177094.137605:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.137609:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177094.137609:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41ad80 x1872934856335872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1786177094.137614:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418a80.
00000020:00000040:0.0:1786177094.137619:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177094.137620:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177094.137622:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177094.137623:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000001:0.0:1786177094.137624:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177094.137627:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:3.0:1786177094.137627:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786177094.137630:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177094.137633:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177094.137633:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000010:0.0:1786177094.137637:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4000.
00000020:00000001:3.0:1786177094.137637:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177094.137638:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:0.0:1786177094.137641:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177094.137642:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177094.137646:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
00000100:00100000:0.0:1786177094.137650:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1786177094.137652:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177094.137655:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177094.137658:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41ad80.
00000100:00000001:0.0:1786177094.137662:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177094.137663:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177094.137666:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.137667:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.137671:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177094.137672:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000606c34cb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177094.137679:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177094.137682:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7327)
00000100:00000001:3.0:1786177094.137686:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177094.137689:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.137694:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177094.137696:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177094.137697:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177094.137701:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.137706:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177094.137707:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177094.137708:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177094.137710:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177094.137711:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177094.137713:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177094.137715:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177094.137717:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177094.137718:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177094.137719:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177094.137722:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177094.137723:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177094.137726:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177094.137728:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177094.137730:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177094.137732:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856336000, portal 10
00000100:00000001:3.0:1786177094.137734:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177094.137735:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761668224 : -114547947883392 : ffff97d1ba41b480)
00000100:00000040:3.0:1786177094.137740:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.137745:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177094.137746:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177094.137748:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177094.137749:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856336000, offset 0
00000400:00000200:3.0:1786177094.137752:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177094.137754:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177094.137771:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177094.137773:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177094.137776:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177094.137777:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177094.137779:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177094.137781:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254796:254796:256:4294967295] 192.168.202.110@tcp LPNI seq info [254785:254785:8:4294967295]
00000400:00000200:3.0:1786177094.137784:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177094.137790:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177094.137793:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177094.137796:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177094.137798:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177094.137800:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786177094.137804:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177094.137805:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1786177094.137808:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177094.137809:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177094.137811:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177094.137814:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177094.137816:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177094.137817:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177094.137819:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177094.137821:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.137831:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177094.137833:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.137838:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.137839:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177094.137841:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177094.137844:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177094.137847:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177094.137848:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301989898
00000100:00000001:3.0:1786177094.137851:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301993677
00000100:00000001:3.0:1786177094.137852:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301989898
00000100:00000001:3.0:1786177094.137854:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301993677
00000100:00000001:3.0:1786177094.137855:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177094.137856:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.137857:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177094.137860:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177094/real 1786177094]  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177094.137865:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177094.137866:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177094.137867:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177094.137868:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177094.137869:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177094.137871:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177094.137873:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177094.137878:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.137882:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177094.137884:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.137885:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177094.137886:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177094.137888:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.137893:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177094.137895:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.137899:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177094.137901:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.137901:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177094.137902:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.137903:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177094.137905:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177094.137906:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177094.137908:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177094.137909:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177094.137912:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177094.137916:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177094.137919:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177094.137921:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.137925:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177094.137926:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177094.137928:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177094.137929:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177094.137930:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177094.137932:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177094.137934:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177094.137936:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177094.137937:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177094.137938:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177094.137940:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177094.137941:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177094.137943:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177094.137945:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177094.137947:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177094.137949:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856336128, portal 10
00000100:00000001:3.0:1786177094.137950:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177094.137951:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761667328 : -114547947884288 : ffff97d1ba41b100)
00000100:00000040:3.0:1786177094.137955:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.137959:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177094.137960:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177094.137961:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177094.137963:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856336128, offset 0
00000400:00000200:3.0:1786177094.137965:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177094.137968:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177094.137970:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177094.137972:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177094.137974:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177094.137975:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177094.137977:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177094.137979:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254797:254797:256:4294967295] 192.168.202.110@tcp LPNI seq info [254786:254786:8:4294967295]
00000400:00000200:3.0:1786177094.137982:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177094.137986:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177094.137989:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177094.137990:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177094.137993:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177094.137995:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786177094.137997:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177094.137999:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1786177094.138002:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177094.138003:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177094.138005:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177094.138007:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177094.138009:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177094.138010:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177094.138011:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177094.138014:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138019:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177094.138021:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138025:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.138026:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177094.138027:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177094.138029:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177094.138031:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177094.138032:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301990023
00000100:00000001:3.0:1786177094.138034:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301993677
00000100:00000001:3.0:1786177094.138035:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301990023
00000100:00000001:3.0:1786177094.138037:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301993677
00000100:00000001:3.0:1786177094.138038:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177094.138040:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.138041:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177094.138043:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177094/real 1786177094]  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177094.138047:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177094.138048:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177094.138049:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177094.138051:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177094.138052:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177094.138053:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177094.138055:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177094.138060:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138064:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177094.138065:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138066:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177094.138067:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177094.138069:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138073:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177094.138075:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138079:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177094.138080:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138081:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177094.138082:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138083:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177094.138085:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177094.138086:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177094.138088:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177094.138090:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177094.138092:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177094.138101:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177094.138103:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177094.138105:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138109:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177094.138110:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177094.138111:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177094.138112:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177094.138114:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177094.138115:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177094.138117:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177094.138118:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177094.138119:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177094.138120:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177094.138122:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177094.138123:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177094.138125:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177094.138127:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177094.138129:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177094.138131:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856336256, portal 4
00000100:00000001:3.0:1786177094.138132:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177094.138133:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761659264 : -114547947892352 : ffff97d1ba419180)
00000100:00000040:3.0:1786177094.138137:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138141:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177094.138142:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177094.138144:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177094.138145:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856336256, offset 0
00000400:00000200:3.0:1786177094.138148:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177094.138152:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177094.138156:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177094.138158:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177094.138161:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177094.138163:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177094.138165:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177094.138168:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254798:254798:256:4294967295] 192.168.202.110@tcp LPNI seq info [254787:254787:8:4294967295]
00000400:00000200:3.0:1786177094.138173:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177094.138179:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177094.138184:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177094.138188:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177094.138192:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177094.138195:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786177094.138199:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177094.138202:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1786177094.138206:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177094.138208:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177094.138210:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177094.138212:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177094.138215:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177094.138216:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177094.138218:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177094.138221:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138226:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177094.138228:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138232:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.138233:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177094.138236:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177094.138238:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177094.138240:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177094.138242:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301989898
00000100:00000001:3.0:1786177094.138244:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301993677
00000100:00000001:3.0:1786177094.138245:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301989898
00000100:00000001:3.0:1786177094.138247:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301993677
00000100:00000001:3.0:1786177094.138248:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177094.138249:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.138250:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177094.138252:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177094/real 1786177094]  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177094.138257:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177094.138259:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177094.138260:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177094.138261:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177094.138263:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177094.138264:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177094.138266:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177094.138271:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138275:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177094.138276:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138277:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177094.138278:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177094.138280:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138284:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177094.138286:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138290:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177094.138292:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138293:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177094.138294:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138295:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177094.138297:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177094.138298:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177094.138300:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177094.138301:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177094.138304:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177094.138308:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177094.138310:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177094.138312:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138316:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177094.138317:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177094.138318:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177094.138320:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177094.138321:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177094.138323:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177094.138325:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177094.138326:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177094.138328:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177094.138329:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177094.138331:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177094.138332:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177094.138334:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177094.138335:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177094.138337:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177094.138339:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856336384, portal 4
00000100:00000001:3.0:1786177094.138341:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177094.138342:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761657472 : -114547947894144 : ffff97d1ba418a80)
00000100:00000040:3.0:1786177094.138345:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138350:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177094.138351:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177094.138352:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177094.138354:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856336384, offset 0
00000400:00000200:3.0:1786177094.138356:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177094.138358:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177094.138360:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177094.138363:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177094.138364:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177094.138366:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177094.138367:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177094.138370:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254799:254799:256:4294967295] 192.168.202.110@tcp LPNI seq info [254788:254788:8:4294967295]
00000400:00000200:3.0:1786177094.138372:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177094.138377:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177094.138379:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177094.138381:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177094.138383:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177094.138386:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786177094.138388:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177094.138390:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1786177094.138392:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177094.138394:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177094.138395:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177094.138397:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177094.138399:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177094.138400:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177094.138402:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177094.138404:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138413:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177094.138415:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177119 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138419:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.138420:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177094.138422:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177094.138423:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177094.138425:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177094.138426:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301989906
00000100:00000001:3.0:1786177094.138428:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301993677
00000100:00000001:3.0:1786177094.138429:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301989906
00000100:00000001:3.0:1786177094.138431:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301993677
00000100:00000001:3.0:1786177094.138433:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177094.138434:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.138435:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177094.138437:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177094/real 1786177094]  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177094.138441:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177094.138442:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177094.138443:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177094.138444:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177094.138445:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177094.138446:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177094.138448:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177094.138453:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138457:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177094.138459:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138460:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177094.138461:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177094.138463:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138467:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177094.138469:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138473:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177094.138475:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138476:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177094.138476:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138477:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177094.138478:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177094.138480:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177094.138482:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177094.138483:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177094.138485:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177094.138490:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177094.138492:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138493:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177094.138495:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b480 x1872934856336000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138499:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177094.138500:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177094.138502:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177094.138503:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177094.138504:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177094.138505:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177094.138507:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177094.138508:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7800.
02000000:00000001:3.0:1786177094.138525:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177094.138527:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177094.138528:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b480.
00000100:00000001:3.0:1786177094.138530:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177094.138531:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138532:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177094.138535:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b100 x1872934856336128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138539:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177094.138540:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177094.138541:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177094.138543:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177094.138543:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177094.138544:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177094.138546:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177094.138547:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:3.0:1786177094.138548:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177094.138549:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177094.138550:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b100.
00000100:00000001:3.0:1786177094.138551:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177094.138551:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138552:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177094.138554:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419180 x1872934856336256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138558:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177094.138558:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177094.138559:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177094.138561:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177094.138561:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177094.138562:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177094.138563:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177094.138564:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5800.
02000000:00000001:3.0:1786177094.138565:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177094.138566:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177094.138567:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419180.
00000100:00000001:3.0:1786177094.138569:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177094.138570:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138571:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177094.138573:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418a80 x1872934856336384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177119 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177094.138577:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177094.138577:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177094.138578:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177094.138579:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177094.138580:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177094.138581:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177094.138582:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177094.138582:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4000.
02000000:00000001:3.0:1786177094.138584:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177094.138584:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177094.138585:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418a80.
00000100:00000001:3.0:1786177094.138587:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177094.138588:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138589:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177094.138591:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177094.138592:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177094.138593:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177094.138594:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177094.712624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177094.712629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177094.712633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177094.712634:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177094.712636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177094.712637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177094.712704:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177094.712710:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177094.712713:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177095.160659:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177095.160665:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177095.160668:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177095.160674:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177095.160678:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177095.160682:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177095.160685:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177095.160688:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177095.160692:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177095.736629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177095.736637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177095.736643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177095.736644:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177095.736648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177095.736649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177095.736735:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177095.736754:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177095.736760:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177096.184631:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177096.184639:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177096.184641:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177096.184648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177096.184652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177096.184655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177096.184658:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177096.184661:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177096.184663:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177096.760685:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177096.760694:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177096.760717:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177096.760718:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177096.760720:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177096.760722:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177096.760797:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177096.760801:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177096.760805:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177097.208650:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177097.208658:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177097.208660:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177097.208667:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177097.208670:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177097.208673:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177097.208676:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177097.208678:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177097.208681:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177097.784653:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177097.784661:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177097.784666:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177097.784668:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177097.784672:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177097.784673:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177097.784781:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177097.784787:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177097.784797:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177097.976626:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301993677, at 4301993728
00000100:00000040:0.0:1786177097.976627:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301993677, at 4301993728
00000100:00000010:1.0:1786177097.976643:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad374e0.
00000100:00000010:0.0:1786177097.976646:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad379c0.
00000100:00000001:3.0:1786177098.232618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177098.232627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177098.232629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177098.232635:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177098.232638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177098.232642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177098.232645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177098.232647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177098.232650:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177098.808616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177098.808621:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177098.808626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177098.808627:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177098.808630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177098.808631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177098.808685:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177098.808691:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177098.808695:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177099.256595:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177099.256604:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177099.256607:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177099.256610:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1786177099.256613:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:0.0:1786177099.256617:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177099.256619:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177099.256621:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177099.256625:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177099.256632:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177099.256632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.256638:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177099.256640:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177099.256642:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7322
00000100:00000001:3.0:1786177099.256644:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177099.256652:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177099.256660:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177099.256663:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177099.256667:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.256670:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177099.256672:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.256675:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177099.256678:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177099.256680:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177099.256682:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177099.256685:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177099.256687:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177099.256689:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177099.256691:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.256693:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177099.256694:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.256696:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177099.256698:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177099.256700:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177099.256702:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177099.256705:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177099.256731:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177099.256734:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177099.256736:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177099.256737:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177099.256748:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418e00.
00000020:00000040:0.0:1786177099.256751:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177099.256755:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177099.256756:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177099.256759:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177099.256761:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177099.256766:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177099.256774:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:0.0:1786177099.256782:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177099.256792:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177099.256801:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177099.256805:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177099.256809:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003428892b] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177099.256827:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177099.256830:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177099.256832:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177099.256834:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177099.256835:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177099.256837:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177099.256839:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7322
00000400:00000200:0.0:1786177099.256843:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177099.256846:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177099.256848:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177099.256849:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1786177099.256851:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177099.256854:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177099.256855:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177099.256859:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177099.256859:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1786177099.256864:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177099.256866:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177099.256867:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177099.256869:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000040:3.0:1786177099.256870:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177099.256871:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177099.256875:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177099.256877:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177099.256880:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.256881:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177099.256885:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.256889:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177099.256894:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:3.0:1786177099.256897:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1786177099.256898:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177099.256901:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177099.256902:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177099.256906:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177099.256907:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177099.256911:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177099.256911:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177099.256914:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177099.256915:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177099.256918:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00100000:3.0:1786177099.256921:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856336512:192.168.202.110@tcp:250:kworker.0
00000008:00100000:0.0:1786177099.256924:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1786177099.256925:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:0.0:1786177099.256929:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177099.256931:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1786177099.256932:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177099.256935:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177099.256938:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418a80.
02000000:00000001:3.0:1786177099.256939:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:0.0:1786177099.256944:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177099.256946:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177099.256948:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177099.256949:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:0.0:1786177099.256950:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177099.256952:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177099.256954:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177099.256956:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177099.256960:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4c00.
00000100:00000001:0.0:1786177099.256964:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177099.256968:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:0.0:1786177099.256971:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177099.256978:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177099.256980:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177099.256982:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000606c34cb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1786177099.256983:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000001:0.0:1786177099.256993:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177099.256995:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177099.256998:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:3.0:1786177099.256998:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177099.257001:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177099.257004:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177099.257006:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177099.257007:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7322
00000100:00000200:3.0:1786177099.257010:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856336512, portal 25
00000400:00000200:0.0:1786177099.257011:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177099.257017:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177099.257017:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1786177099.257021:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177099.257021:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761658368 : -114547947893248 : ffff97d1ba418e00)
00000100:00000040:0.0:1786177099.257026:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.257029:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177099.257032:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177099.257033:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.257037:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177099.257042:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177099.257044:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177099.257046:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177099.257049:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000001:3.0:1786177099.257050:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:0.0:1786177099.257052:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177099.257055:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177099.257057:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:0.0:1786177099.257060:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:3.0:1786177099.257061:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000001:0.0:1786177099.257063:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177099.257066:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:3.0:1786177099.257067:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856336512, offset 0
00000100:00000001:0.0:1786177099.257069:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177099.257073:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786177099.257075:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177099.257079:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177099.257080:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786177099.257081:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177099.257085:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177099.257086:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177099.257088:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177099.257089:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177099.257092:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177099.257092:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177099.257094:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177099.257098:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419180.
00000400:00000200:3.0:1786177099.257101:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:0.0:1786177099.257102:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177099.257107:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177099.257109:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177099.257110:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1786177099.257111:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1786177099.257112:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177099.257116:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177099.257119:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000400:00000200:3.0:1786177099.257119:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1786177099.257122:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177099.257126:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00100000:0.0:1786177099.257130:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177099.257130:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177099.257138:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254800:254800:256:4294967295] 192.168.202.110@tcp LPNI seq info [254789:254789:8:4294967295]
00000100:00000001:0.0:1786177099.257145:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177099.257148:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177099.257149:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177099.257156:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005a89d8ab] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177099.257167:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177099.257169:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177099.257173:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177099.257176:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177099.257178:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:3.0:1786177099.257179:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177099.257180:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177099.257185:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177099.257188:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7322
00000400:00000200:0.0:1786177099.257190:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177099.257192:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177099.257193:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000010:3.0:1786177099.257195:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000040:0.0:1786177099.257196:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.257199:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177099.257200:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.257202:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.0:1786177099.257206:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:0.0:1786177099.257207:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177099.257210:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177099.257213:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:3.0:1786177099.257213:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000020:00000040:0.0:1786177099.257215:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177099.257220:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177099.257224:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000200:3.1:1786177099.257225:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786177099.257229:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:3.1:1786177099.257231:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:0.0:1786177099.257232:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177099.257237:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.257239:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000100:3.0:1786177099.257241:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:0.0:1786177099.257242:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000800:00000010:3.0:1786177099.257245:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00080000:0.0:1786177099.257247:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177099.257250:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177099.257252:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177099.257254:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177099.257255:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:0.0:1786177099.257256:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177099.257261:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177099.257263:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:3.0:1786177099.257265:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000008:00000001:0.0:1786177099.257266:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177099.257270:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177099.257272:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000100:00000010:0.0:1786177099.257274:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b100.
00000400:00000010:3.0:1786177099.257275:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000020:00000040:0.0:1786177099.257277:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177099.257280:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177099.257281:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.257282:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1786177099.257283:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177099.257286:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177099.257287:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177099.257289:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786177099.257290:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000100:00000001:0.0:1786177099.257295:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177099.257302:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177099.257305:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177099.257310:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177099.257312:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177099.257312:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177099.257321:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000fa632fb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.257327:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177099.257332:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.257332:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:0.0:1786177099.257336:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177099.257338:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177099.257339:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177099.257339:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177099.257340:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177099.257345:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177099.257347:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177099.257349:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7322
00000100:00000001:3.0:1786177099.257351:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4301998797
00000400:00000200:0.0:1786177099.257355:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177099.257358:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000010:3.0:1786177099.257359:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37420.
00000100:00080000:0.0:1786177099.257360:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177099.257365:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.257366:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177099.257368:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.257369:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177099.257371:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177099.257371:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000040:0.0:1786177099.257374:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177099.257376:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177099.257378:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:3.0:1786177099.257379:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4301998797
00000020:00000001:0.0:1786177099.257382:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000010:3.0:1786177099.257384:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37180.
00000020:00000001:0.0:1786177099.257385:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:3.0:1786177099.257389:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:0.0:1786177099.257390:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.257394:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177099.257396:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1786177099.257397:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177099.257401:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177099.257401:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177099.257407:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:3.0:1786177099.257407:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00080000:0.0:1786177099.257412:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177099.257416:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177099.257417:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177099/real 1786177099]  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177099.257420:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177099.257423:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177099.257428:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177099.257432:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177099.257435:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177099.257441:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177099.257443:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1786177099.257444:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177099.257446:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000010:0.0:1786177099.257449:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b480.
00000400:00000010:3.0:1786177099.257450:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000020:00000040:0.0:1786177099.257452:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000400:00000200:3.0:1786177099.257454:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
02000000:00000001:0.0:1786177099.257456:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:3.0:1786177099.257457:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
02000000:00000001:0.0:1786177099.257458:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177099.257462:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177099.257462:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:0.0:1786177099.257463:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177099.257466:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177099.257468:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786177099.257470:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
00000100:00000001:0.0:1786177099.257474:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177099.257482:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1786177099.257485:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177099.257505:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177099.257507:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177099.257508:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.257528:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.257531:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177099.257533:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177099.257542:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001215e9e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177099.257542:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177099.257569:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.257571:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177099.257577:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7332)
00000100:00000040:3.0:1786177099.257578:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.257586:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177099.257588:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.257589:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177099.257590:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.257592:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177099.257593:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177099.257596:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177099.257598:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177099.257601:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177099.257604:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177099.257612:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856336512:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177099.257616:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.257618:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177099.257620:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418e00 x1872934856336512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.257624:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177099.257626:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177099.257628:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177099.257631:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177099.257632:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177099.257634:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177099.257635:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177099.257637:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:3.0:1786177099.257640:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177099.257642:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177099.257643:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418e00.
00000100:00000001:3.0:1786177099.257647:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177099.257648:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.257650:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.257657:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177099.257658:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.257659:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177099.257661:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177099.257662:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177099.257664:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.257670:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177099.257671:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177099.257673:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177099.257674:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177099.257675:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177099.257677:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177099.257680:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177099.257681:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177099.257682:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177099.257684:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177099.257686:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177099.257687:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177099.257689:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177099.257691:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177099.257694:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177099.257696:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856336640, portal 10
00000100:00000001:3.0:1786177099.257698:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177099.257699:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761657472 : -114547947894144 : ffff97d1ba418a80)
00000100:00000040:3.0:1786177099.257704:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.257723:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177099.257725:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177099.257727:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177099.257729:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856336640, offset 0
00000400:00000200:3.0:1786177099.257732:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177099.257735:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177099.257738:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177099.257740:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177099.257742:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177099.257744:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177099.257746:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177099.257748:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254801:254801:256:4294967295] 192.168.202.110@tcp LPNI seq info [254790:254790:8:4294967295]
00000400:00000200:3.0:1786177099.257751:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177099.257755:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177099.257758:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177099.257760:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177099.257763:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177099.257766:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786177099.257769:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177099.257770:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1786177099.257773:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177099.257774:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177099.257777:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177099.257779:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177099.257782:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177099.257783:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177099.257785:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177099.257787:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.257793:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177099.257795:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.257800:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.257801:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177099.257803:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177099.257806:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177099.257808:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177099.257810:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301995018
00000100:00000001:3.0:1786177099.257812:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301998797
00000100:00000001:3.0:1786177099.257814:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301995018
00000100:00000001:3.0:1786177099.257815:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301998797
00000100:00000001:3.0:1786177099.257817:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177099.257818:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.257819:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177099.257822:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177099/real 1786177099]  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177099.257826:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177099.257827:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177099.257829:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177099.257831:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177099.257832:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177099.257833:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177099.257835:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177099.257841:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.257844:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177099.257845:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.257846:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177099.257847:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177099.257849:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.257853:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177099.257856:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.257861:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177099.257863:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.257864:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177099.257865:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.257866:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177099.257867:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177099.257869:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177099.257870:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177099.257872:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177099.257874:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177099.257879:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177099.257882:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177099.257884:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.257889:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177099.257890:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177099.257891:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177099.257892:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177099.257894:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177099.257895:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177099.257897:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177099.257898:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177099.257899:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177099.257901:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177099.257903:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177099.257904:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177099.257906:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177099.257908:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177099.257910:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177099.257912:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856336768, portal 10
00000100:00000001:3.0:1786177099.257913:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177099.257914:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761659264 : -114547947892352 : ffff97d1ba419180)
00000100:00000040:3.0:1786177099.257919:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.257923:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177099.257925:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177099.257926:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177099.257928:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856336768, offset 0
00000400:00000200:3.0:1786177099.257930:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177099.257932:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177099.257934:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177099.257936:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177099.257938:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177099.257940:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177099.257941:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177099.257944:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254802:254802:256:4294967295] 192.168.202.110@tcp LPNI seq info [254791:254791:8:4294967295]
00000400:00000200:3.0:1786177099.257946:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177099.257952:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177099.257957:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177099.257960:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177099.257964:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177099.257966:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786177099.257971:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177099.257974:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1786177099.257978:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177099.257980:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177099.257983:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177099.257986:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177099.257990:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177099.257992:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177099.257994:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177099.257999:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258006:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177099.258009:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258014:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.258015:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177099.258018:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177099.258020:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177099.258022:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177099.258023:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301995143
00000100:00000001:3.0:1786177099.258025:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301998797
00000100:00000001:3.0:1786177099.258027:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301995143
00000100:00000001:3.0:1786177099.258029:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301998797
00000100:00000001:3.0:1786177099.258031:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177099.258032:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.258034:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177099.258036:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177099/real 1786177099]  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177099.258041:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177099.258042:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177099.258043:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177099.258044:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177099.258045:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177099.258046:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177099.258048:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177099.258054:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258058:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177099.258059:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258060:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177099.258061:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177099.258063:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258067:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177099.258069:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258074:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177099.258075:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258076:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177099.258077:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258078:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177099.258080:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177099.258082:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177099.258084:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177099.258086:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177099.258088:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177099.258093:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177099.258096:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177099.258098:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258102:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177099.258103:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177099.258105:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177099.258106:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177099.258107:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177099.258109:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177099.258110:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177099.258112:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177099.258113:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177099.258114:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177099.258116:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177099.258118:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177099.258121:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177099.258122:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177099.258125:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177099.258127:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856336896, portal 4
00000100:00000001:3.0:1786177099.258128:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177099.258129:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761667328 : -114547947884288 : ffff97d1ba41b100)
00000100:00000040:3.0:1786177099.258134:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258139:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177099.258141:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177099.258142:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177099.258144:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856336896, offset 0
00000400:00000200:3.0:1786177099.258146:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177099.258149:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177099.258151:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177099.258153:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177099.258154:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177099.258156:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177099.258158:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177099.258160:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254803:254803:256:4294967295] 192.168.202.110@tcp LPNI seq info [254792:254792:8:4294967295]
00000400:00000200:3.0:1786177099.258163:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177099.258167:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177099.258171:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177099.258173:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177099.258175:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177099.258177:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786177099.258180:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177099.258182:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1786177099.258185:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177099.258186:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177099.258188:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177099.258191:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177099.258192:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177099.258194:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177099.258195:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177099.258198:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258203:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177099.258205:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258214:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.258215:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177099.258217:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177099.258219:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177099.258221:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177099.258222:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301995018
00000100:00000001:3.0:1786177099.258224:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301998797
00000100:00000001:3.0:1786177099.258226:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301995018
00000100:00000001:3.0:1786177099.258227:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301998797
00000100:00000001:3.0:1786177099.258228:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177099.258230:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.258231:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177099.258233:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177099/real 1786177099]  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177099.258237:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177099.258238:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177099.258239:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177099.258240:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177099.258241:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177099.258242:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177099.258244:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177099.258250:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258254:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177099.258254:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258255:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177099.258256:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177099.258258:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258263:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177099.258266:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258270:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177099.258272:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258273:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177099.258273:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258274:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177099.258275:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177099.258277:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177099.258279:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177099.258280:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177099.258282:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177099.258287:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856336896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177099.258289:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177099.258291:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258294:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177099.258295:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177099.258296:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177099.258297:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177099.258299:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177099.258300:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856337024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177099.258302:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177099.258303:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177099.258305:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177099.258306:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177099.258308:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177099.258309:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177099.258311:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177099.258313:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177099.258315:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177099.258317:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856337024, portal 4
00000100:00000001:3.0:1786177099.258318:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177099.258319:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761668224 : -114547947883392 : ffff97d1ba41b480)
00000100:00000040:3.0:1786177099.258323:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258326:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177099.258328:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177099.258329:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177099.258331:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856337024, offset 0
00000400:00000200:3.0:1786177099.258333:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177099.258335:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177099.258337:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177099.258339:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177099.258340:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177099.258342:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177099.258343:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177099.258346:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254804:254804:256:4294967295] 192.168.202.110@tcp LPNI seq info [254793:254793:8:4294967295]
00000400:00000200:3.0:1786177099.258348:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177099.258352:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177099.258355:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177099.258357:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177099.258359:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177099.258361:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786177099.258364:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177099.258365:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1786177099.258368:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177099.258370:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177099.258372:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177099.258374:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177099.258376:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177099.258377:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177099.258378:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177099.258380:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258385:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177099.258387:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258391:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.258392:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177099.258394:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177099.258396:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177099.258398:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177099.258399:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4301995026
00000100:00000001:3.0:1786177099.258401:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4301998797
00000100:00000001:3.0:1786177099.258402:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4301995026
00000100:00000001:3.0:1786177099.258404:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4301998797
00000100:00000001:3.0:1786177099.258405:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177099.258406:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.258407:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177099.258410:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177099/real 1786177099]  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177099.258414:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177099.258415:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177099.258415:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177099.258417:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177099.258418:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177099.258419:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177099.258422:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177099.258427:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258431:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177099.258432:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258433:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177099.258434:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177099.258436:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258440:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177099.258442:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258446:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177099.258447:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258448:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177099.258449:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258450:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177099.258451:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177099.258453:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177099.258454:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177099.258455:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177099.258458:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177099.258462:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856337024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177099.258464:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258466:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177099.258468:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418a80 x1872934856336640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258473:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177099.258474:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177099.258476:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177099.258477:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177099.258478:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177099.258479:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177099.258480:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177099.258481:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:3.0:1786177099.258483:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177099.258484:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177099.258485:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418a80.
00000100:00000001:3.0:1786177099.258487:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177099.258488:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258490:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177099.258492:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419180 x1872934856336768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258496:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177099.258497:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177099.258498:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177099.258499:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177099.258500:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177099.258502:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177099.258503:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177099.258504:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:3.0:1786177099.258505:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177099.258506:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177099.258507:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419180.
00000100:00000001:3.0:1786177099.258508:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177099.258509:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258525:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177099.258527:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b100 x1872934856336896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258531:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177099.258533:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177099.258534:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177099.258536:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177099.258537:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177099.258538:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177099.258539:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177099.258540:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:3.0:1786177099.258542:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177099.258542:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177099.258543:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b100.
00000100:00000001:3.0:1786177099.258545:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177099.258545:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258547:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177099.258549:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b480 x1872934856337024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177124 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177099.258553:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177099.258554:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177099.258555:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177099.258556:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177099.258557:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177099.258558:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177099.258559:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177099.258559:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
02000000:00000001:3.0:1786177099.258561:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177099.258561:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177099.258562:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b480.
00000100:00000001:3.0:1786177099.258563:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177099.258564:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258565:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177099.258567:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177099.258568:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177099.258569:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177099.258570:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177099.832616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177099.832621:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177099.832625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177099.832626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177099.832628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177099.832629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177099.832781:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177099.832789:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177099.832794:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177100.280586:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177100.280592:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177100.280609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177100.280614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177100.280618:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177100.280623:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177100.280626:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177100.280629:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177100.280632:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177100.856619:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177100.856627:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177100.856629:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177100.856632:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177100.856636:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177100.856639:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177100.856640:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177100.856642:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177100.856645:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177100.856647:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177100.856648:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177100.856649:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177100.856651:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177100.856652:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177100.856653:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177100.856654:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177100.856657:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177100.856658:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177100.856762:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177100.856768:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177100.856771:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177101.304604:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177101.304613:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177101.304615:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177101.304621:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177101.304625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177101.304629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177101.304632:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177101.304634:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177101.304636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177101.880632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177101.880637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177101.880641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177101.880642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177101.880644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177101.880645:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177101.880745:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177101.880751:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177101.880755:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177102.328615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177102.328622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177102.328623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177102.328628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177102.328630:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177102.328635:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177102.328637:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177102.328639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177102.328641:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177102.904609:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177102.904614:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177102.904617:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177102.904618:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177102.904621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177102.904623:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177102.904624:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177102.904627:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177102.904630:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177102.904632:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177102.904633:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177102.904635:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177102.904700:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177102.904704:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177102.904707:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177103.096606:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4301998797, at 4301998848
00000100:00000040:1.0:1786177103.096606:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4301998797, at 4301998848
00000100:00000010:1.0:1786177103.096621:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37180.
00000100:00000010:0.0:1786177103.096623:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37420.
00000100:00000001:3.0:1786177103.352586:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177103.352592:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177103.352594:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177103.352598:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177103.352601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177103.352604:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177103.352606:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177103.352607:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177103.352609:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177103.928628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177103.928633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177103.928638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177103.928639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177103.928641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177103.928642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177103.928719:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177103.928724:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177103.928728:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177104.376601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177104.376608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177104.376610:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177104.376614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177104.376618:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177104.376621:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.376624:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177104.376627:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177104.376629:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177104.376629:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177104.376636:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177104.376639:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177104.376642:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177104.376645:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177104.376652:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7327
00000400:00000200:2.0:1786177104.376674:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177104.376681:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177104.376683:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177104.376686:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.376688:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177104.376690:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.376692:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177104.376694:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177104.376696:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177104.376698:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177104.376700:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177104.376702:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177104.376704:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177104.376706:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.376708:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177104.376710:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.376711:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177104.376713:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177104.376715:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177104.376717:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177104.376720:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177104.376723:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177104.376725:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177104.376726:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177104.376728:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177104.376736:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786177104.376738:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177104.376742:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177104.376743:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177104.376746:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177104.376748:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177104.376752:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177104.376758:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786177104.376763:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177104.376771:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177104.376779:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177104.376782:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177104.376786:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177104.376801:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177104.376804:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177104.376806:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177104.376807:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177104.376808:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177104.376811:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177104.376813:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7327
00000400:00000200:2.0:1786177104.376817:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177104.376819:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177104.376821:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177104.376824:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177104.376824:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177104.376826:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177104.376829:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1786177104.376830:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.376833:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177104.376834:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1786177104.376835:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177104.376838:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177104.376840:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177104.376841:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177104.376844:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177104.376846:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177104.376846:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177104.376852:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.376855:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177104.376859:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.376864:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177104.376869:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:3.0:1786177104.376872:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1786177104.376873:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177104.376876:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177104.376877:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177104.376882:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177104.376885:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177104.376886:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177104.376888:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177104.376890:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177104.376893:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177104.376894:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00100000:3.0:1786177104.376900:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856337152:192.168.202.110@tcp:250:kworker.0
00000008:00100000:2.0:1786177104.376901:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1786177104.376906:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1786177104.376908:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177104.376911:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1786177104.376912:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177104.376914:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177104.376917:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:2.0:1786177104.376919:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
02000000:00000010:3.0:1786177104.376923:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
00000020:00000040:2.0:1786177104.376924:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177104.376929:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177104.376931:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177104.376934:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177104.376935:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177104.376940:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1786177104.376941:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
02000000:00000001:2.0:1786177104.376943:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177104.376948:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177104.376948:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
02000000:00000010:2.0:1786177104.376953:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000200:3.0:1786177104.376956:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856337152, portal 25
00000100:00000001:2.0:1786177104.376957:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.376961:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177104.376963:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00100000:2.0:1786177104.376967:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1786177104.376973:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177104.376978:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177104.376982:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177104.376989:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.376993:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786177104.377000:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177104.377003:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786177104.377004:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177104.377008:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177104.377009:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177104.377011:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177104.377012:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177104.377015:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7327
00000400:00000200:2.0:1786177104.377018:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177104.377020:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177104.377021:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1786177104.377022:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000040:2.0:1786177104.377024:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.377028:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177104.377030:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:3.0:1786177104.377031:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856337152, offset 0
00000100:00000001:2.0:1786177104.377032:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177104.377036:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177104.377038:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177104.377041:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:3.0:1786177104.377041:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:2.0:1786177104.377045:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177104.377049:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177104.377054:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1786177104.377056:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177104.377062:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1786177104.377064:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.377069:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177104.377069:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1786177104.377075:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177104.377078:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1786177104.377080:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177104.377086:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:3.0:1786177104.377087:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1786177104.377092:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1786177104.377095:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.377096:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177104.377099:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177104.377102:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177104.377103:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254805:254805:256:4294967295] 192.168.202.110@tcp LPNI seq info [254794:254794:8:4294967295]
02000000:00000001:2.0:1786177104.377105:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177104.377110:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177104.377114:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:3.0:1786177104.377118:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:2.0:1786177104.377119:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177104.377122:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177104.377126:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786177104.377128:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177104.377132:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177104.377135:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177104.377136:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786177104.377137:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177104.377140:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177104.377144:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177104.377150:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000010:2.0:1786177104.377151:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000800:00000010:3.0:1786177104.377157:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000001:2.0:1786177104.377158:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177104.377168:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.0:1786177104.377170:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177104.377178:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000100:00000001:2.0:1786177104.377183:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177104.377188:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177104.377189:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1786177104.377195:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1786177104.377196:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1786177104.377208:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177104.377213:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00000001:2.0:1786177104.377214:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177104.377218:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1786177104.377221:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:2.0:1786177104.377222:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177104.377225:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177104.377228:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1786177104.377229:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1786177104.377231:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177104.377235:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7327
00000400:00000200:3.0:1786177104.377235:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177104.377239:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000400:00000200:2.0:1786177104.377241:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177104.377245:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:2.0:1786177104.377246:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177104.377249:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:3.0:1786177104.377253:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177104.377254:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.377259:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177104.377262:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.377266:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177104.377270:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177104.377272:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177104.377274:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177104.377278:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177104.377280:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177104.377282:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:3.0:1786177104.377282:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177104.377287:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.377290:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177104.377292:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177104.377294:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.377297:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177104.377298:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177104.377300:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177104.377302:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177104.377303:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177104.377305:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177104.377306:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177104.377310:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177104.377314:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177104.377318:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.377318:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177104.377323:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.377323:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000010:2.0:1786177104.377328:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786177104.377330:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177104.377332:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177104.377333:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177104.377337:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177104.377340:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1786177104.377340:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177104.377342:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177104.377344:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302003917
02000000:00000001:2.0:1786177104.377345:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177104.377353:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000010:3.0:1786177104.377354:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37f00.
00000100:00000001:2.0:1786177104.377356:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177104.377363:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177104.377363:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177104.377370:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302003917
00000100:00000001:2.0:1786177104.377372:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177104.377376:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177104.377376:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37a20.
00000100:00000001:3.0:1786177104.377380:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:2.0:1786177104.377381:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.377385:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177104.377390:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177104.377396:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.377396:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00080000:2.0:1786177104.377400:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177104.377405:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00100000:3.0:1786177104.377405:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177104/real 1786177104]  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177104.377408:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177104.377411:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177104.377416:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177104.377421:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7327
00000400:00000001:3.0:1786177104.377423:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:2.0:1786177104.377425:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000001:3.0:1786177104.377426:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177104.377429:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:2.0:1786177104.377430:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177104.377434:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1786177104.377434:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177104.377438:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000040:2.0:1786177104.377440:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177104.377442:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1786177104.377443:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1786177104.377448:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177104.377449:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.377454:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177104.377459:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177104.377461:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177104.377464:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177104.377467:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000200:3.0:1786177104.377468:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177104.377469:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786177104.377474:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177104.377481:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.377484:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177104.377489:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:2.0:1786177104.377490:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177104.377494:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177104.377495:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786177104.377496:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:3.0:1786177104.377501:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:2.0:1786177104.377502:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177104.377506:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.377507:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786177104.377508:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177104.377536:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177104.377538:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177104.377543:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177104.377545:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177104.377548:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177104.377548:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177104.377552:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177104.377557:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000100:00000001:3.0:1786177104.377560:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786177104.377561:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177104.377564:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177104.377566:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177104.377566:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177104.377570:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177104.377573:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177104.377575:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.377581:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000010:2.0:1786177104.377583:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:3.0:1786177104.377585:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.377589:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177104.377590:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177104.377591:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.377592:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177104.377595:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177104.377599:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00100000:2.0:1786177104.377603:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00080000:3.0:1786177104.377605:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177104.377610:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177104.377618:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177104.377639:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856337152:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177104.377643:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177104.377644:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177104.377646:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.377648:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177104.377654:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177104.377655:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856337152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177104.377675:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177104.377682:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7337)
00000100:00000001:3.0:1786177104.377683:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177104.377688:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177104.377690:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177104.377693:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177104.377695:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177104.377697:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177104.377699:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177104.377701:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:3.0:1786177104.377709:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177104.377710:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177104.377712:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:3.0:1786177104.377714:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177104.377715:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.377717:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.377723:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177104.377724:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.377725:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177104.377727:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177104.377728:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177104.377731:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.377736:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177104.377737:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177104.377739:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177104.377740:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177104.377742:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177104.377743:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856337280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177104.377746:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177104.377747:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177104.377748:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177104.377749:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177104.377752:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177104.377753:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177104.377756:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177104.377759:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177104.377763:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177104.377766:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856337280, portal 10
00000100:00000001:3.0:1786177104.377768:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177104.377769:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:3.0:1786177104.377776:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.377784:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177104.377786:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177104.377789:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177104.377792:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856337280, offset 0
00000400:00000200:3.0:1786177104.377797:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177104.377801:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177104.377805:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177104.377807:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177104.377809:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177104.377811:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177104.377813:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177104.377815:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254806:254806:256:4294967295] 192.168.202.110@tcp LPNI seq info [254795:254795:8:4294967295]
00000400:00000200:3.0:1786177104.377818:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177104.377823:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177104.377827:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177104.377830:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177104.377834:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177104.377837:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786177104.377842:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177104.377845:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1786177104.377848:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177104.377850:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177104.377854:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177104.377857:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177104.377861:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177104.377863:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177104.377865:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177104.377870:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.377879:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177104.377882:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.377889:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.377890:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177104.377893:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177104.377897:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177104.377899:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177104.377901:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302000138
00000100:00000001:3.0:1786177104.377903:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302003917
00000100:00000001:3.0:1786177104.377905:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302000138
00000100:00000001:3.0:1786177104.377906:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302003917
00000100:00000001:3.0:1786177104.377908:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177104.377909:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.377911:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177104.377913:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177104/real 1786177104]  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177104.377919:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177104.377920:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177104.377921:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177104.377923:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177104.377924:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177104.377926:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177104.377928:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177104.377933:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.377937:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177104.377938:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.377939:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177104.377940:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177104.377942:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.377946:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177104.377949:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.377953:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177104.377954:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.377955:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177104.377956:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.377957:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177104.377959:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177104.377961:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177104.377963:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177104.377964:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177104.377967:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177104.377972:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856337280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177104.377974:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177104.377978:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.377984:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177104.377986:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177104.377988:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177104.377989:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177104.377991:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177104.377994:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856337408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177104.377998:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177104.378000:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177104.378002:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177104.378004:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177104.378008:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177104.378010:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177104.378014:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177104.378016:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177104.378020:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177104.378023:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856337408, portal 10
00000100:00000001:3.0:1786177104.378025:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177104.378026:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:3.0:1786177104.378032:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378040:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177104.378042:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177104.378043:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177104.378045:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856337408, offset 0
00000400:00000200:3.0:1786177104.378048:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177104.378051:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177104.378053:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177104.378055:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177104.378057:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177104.378059:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177104.378061:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177104.378063:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254807:254807:256:4294967295] 192.168.202.110@tcp LPNI seq info [254796:254796:8:4294967295]
00000400:00000200:3.0:1786177104.378066:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177104.378070:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177104.378073:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177104.378076:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177104.378080:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177104.378083:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786177104.378088:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177104.378091:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1786177104.378094:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177104.378096:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177104.378100:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177104.378104:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177104.378107:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177104.378109:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177104.378111:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177104.378115:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378124:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177104.378127:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378136:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.378137:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177104.378140:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177104.378142:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177104.378145:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177104.378147:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302000263
00000100:00000001:3.0:1786177104.378150:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302003917
00000100:00000001:3.0:1786177104.378152:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302000263
00000100:00000001:3.0:1786177104.378154:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302003917
00000100:00000001:3.0:1786177104.378156:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177104.378159:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.378161:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177104.378164:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177104/real 1786177104]  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177104.378173:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177104.378175:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177104.378177:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177104.378179:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177104.378181:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177104.378183:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177104.378187:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177104.378196:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378200:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177104.378201:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378203:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177104.378204:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177104.378206:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378211:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177104.378214:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378218:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177104.378220:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378221:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177104.378222:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378223:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177104.378224:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177104.378225:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177104.378227:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177104.378229:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177104.378231:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177104.378236:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856337408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177104.378238:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177104.378241:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378245:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177104.378246:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177104.378247:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177104.378249:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177104.378250:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177104.378252:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856337536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177104.378254:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177104.378255:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177104.378257:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177104.378259:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177104.378263:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177104.378265:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177104.378269:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177104.378272:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177104.378276:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177104.378279:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856337536, portal 4
00000100:00000001:3.0:1786177104.378281:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177104.378283:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:3.0:1786177104.378289:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378296:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177104.378299:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177104.378301:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177104.378304:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856337536, offset 0
00000400:00000200:3.0:1786177104.378309:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177104.378312:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177104.378315:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177104.378317:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177104.378319:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177104.378321:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177104.378322:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177104.378325:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254808:254808:256:4294967295] 192.168.202.110@tcp LPNI seq info [254797:254797:8:4294967295]
00000400:00000200:3.0:1786177104.378327:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177104.378332:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177104.378335:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177104.378337:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177104.378339:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177104.378341:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786177104.378344:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177104.378346:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1786177104.378350:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177104.378353:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177104.378356:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177104.378360:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177104.378362:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177104.378364:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177104.378366:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177104.378370:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378377:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177104.378380:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378388:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.378389:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177104.378393:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177104.378396:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177104.378400:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177104.378402:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302000138
00000100:00000001:3.0:1786177104.378404:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302003917
00000100:00000001:3.0:1786177104.378406:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302000138
00000100:00000001:3.0:1786177104.378409:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302003917
00000100:00000001:3.0:1786177104.378411:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177104.378413:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.378415:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177104.378418:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177104/real 1786177104]  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177104.378425:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177104.378427:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177104.378428:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177104.378431:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177104.378432:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177104.378435:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177104.378438:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177104.378446:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378452:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177104.378453:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378455:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177104.378457:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177104.378461:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378468:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177104.378472:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378479:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177104.378482:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378483:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177104.378485:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378486:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177104.378488:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177104.378491:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177104.378494:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177104.378496:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177104.378500:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177104.378528:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856337536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177104.378534:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177104.378538:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378546:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177104.378547:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177104.378550:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177104.378552:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177104.378554:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177104.378557:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856337664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177104.378560:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177104.378562:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177104.378564:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177104.378566:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177104.378570:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177104.378571:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177104.378574:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177104.378575:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177104.378578:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177104.378580:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856337664, portal 4
00000100:00000001:3.0:1786177104.378581:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177104.378582:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00000040:3.0:1786177104.378586:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378591:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177104.378592:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177104.378594:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177104.378595:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856337664, offset 0
00000400:00000200:3.0:1786177104.378598:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177104.378600:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177104.378603:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177104.378605:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177104.378606:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177104.378608:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177104.378610:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177104.378612:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254809:254809:256:4294967295] 192.168.202.110@tcp LPNI seq info [254798:254798:8:4294967295]
00000400:00000200:3.0:1786177104.378615:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177104.378619:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177104.378622:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177104.378625:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177104.378627:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177104.378629:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1786177104.378632:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177104.378634:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1786177104.378637:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177104.378638:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177104.378640:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177104.378642:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177104.378644:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177104.378645:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177104.378647:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177104.378649:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378655:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177104.378656:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378677:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.378678:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177104.378681:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177104.378683:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177104.378685:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177104.378687:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302000147
00000100:00000001:3.0:1786177104.378688:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302003917
00000100:00000001:3.0:1786177104.378690:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302000147
00000100:00000001:3.0:1786177104.378691:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302003917
00000100:00000001:3.0:1786177104.378693:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177104.378694:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.378696:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177104.378698:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177104/real 1786177104]  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177104.378704:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177104.378705:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177104.378706:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177104.378707:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177104.378708:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177104.378710:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177104.378712:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177104.378717:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378721:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177104.378722:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378723:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177104.378724:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177104.378726:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378730:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177104.378733:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378737:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177104.378738:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378739:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177104.378740:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378741:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177104.378742:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177104.378744:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177104.378746:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177104.378747:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177104.378750:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177104.378754:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856337664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177104.378757:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378758:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177104.378761:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856337280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378767:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177104.378769:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177104.378770:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177104.378773:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177104.378774:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177104.378776:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177104.378778:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177104.378780:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3400.
02000000:00000001:3.0:1786177104.378783:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177104.378784:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177104.378786:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:3.0:1786177104.378789:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177104.378790:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378792:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177104.378806:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856337408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378818:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177104.378819:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177104.378821:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177104.378824:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177104.378825:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177104.378826:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177104.378828:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177104.378830:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:3.0:1786177104.378832:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177104.378834:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177104.378835:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:3.0:1786177104.378838:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177104.378840:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378841:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177104.378845:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856337536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378852:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177104.378853:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177104.378855:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177104.378857:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177104.378859:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177104.378860:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177104.378862:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177104.378864:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:3.0:1786177104.378866:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177104.378867:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177104.378868:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:3.0:1786177104.378870:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177104.378871:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378873:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177104.378876:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856337664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177104.378881:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177104.378883:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177104.378885:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177104.378887:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177104.378888:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177104.378889:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177104.378891:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177104.378892:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0800.
02000000:00000001:3.0:1786177104.378895:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177104.378896:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177104.378898:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:3.0:1786177104.378900:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177104.378901:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378903:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177104.378908:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177104.378909:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177104.378911:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177104.378913:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177104.952613:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177104.952618:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177104.952622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177104.952623:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177104.952625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177104.952626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177104.952692:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177104.952695:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177104.952698:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177105.400588:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177105.400597:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177105.400600:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177105.400606:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177105.400610:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177105.400613:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177105.400616:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177105.400619:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177105.400621:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177105.976625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177105.976632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177105.976638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177105.976640:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177105.976654:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177105.976655:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177105.976729:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177105.976733:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177105.976737:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.0:1786177106.296686:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786177106.296695:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786177106.296701:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:1.0:1786177106.296879:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1786177106.296884:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1786177106.296894:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1786177106.296899:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1786177106.296901:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177106.424616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177106.424623:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177106.424624:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177106.424628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177106.424631:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177106.424634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177106.424635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177106.424649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177106.424652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177107.000623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177107.000629:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177107.000646:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177107.000647:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177107.000650:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177107.000651:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177107.000700:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177107.000703:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177107.000706:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177107.448666:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177107.448672:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177107.448674:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177107.448679:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177107.448682:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177107.448685:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177107.448688:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177107.448690:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177107.448692:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177108.024685:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177108.024694:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177108.024701:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177108.024702:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177108.024705:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177108.024706:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177108.024768:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177108.024773:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177108.024776:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177108.216637:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302003917, at 4302003968
00000100:00000040:0.0:1786177108.216643:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302003917, at 4302003968
00000100:00000010:1.0:1786177108.216652:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37a20.
00000100:00000010:0.0:1786177108.216654:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37f00.
00000100:00000001:3.0:1786177108.472602:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177108.472608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177108.472610:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177108.472614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177108.472617:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177108.472632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177108.472634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177108.472636:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177108.472638:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177109.048690:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177109.048700:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177109.048709:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177109.048710:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177109.048714:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177109.048715:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177109.048795:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177109.048801:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177109.048805:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177109.496649:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:0.0:1786177109.496666:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177109.496667:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177109.496670:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177109.496678:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:3.0:1786177109.496679:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177109.496682:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177109.496687:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1786177109.496688:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177109.496693:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177109.496695:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177109.496699:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177109.496706:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177109.496707:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7332
00000100:00000001:3.0:1786177109.496712:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177109.496721:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177109.496731:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177109.496735:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177109.496739:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177109.496744:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177109.496747:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177109.496751:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177109.496756:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177109.496758:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177109.496762:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177109.496766:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177109.496769:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177109.496772:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177109.496776:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177109.496778:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177109.496780:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177109.496782:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177109.496786:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177109.496791:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177109.496794:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177109.496799:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177109.496803:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177109.496806:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177109.496809:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177109.496812:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177109.496823:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41ad80.
00000020:00000040:0.0:1786177109.496828:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177109.496833:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177109.496835:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177109.496839:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177109.496843:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177109.496859:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177109.496871:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000100:00000001:0.0:1786177109.496881:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177109.496892:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177109.496905:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177109.496910:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177109.496916:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034e2c676] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177109.496941:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177109.496945:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177109.496949:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177109.496952:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177109.496954:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177109.496957:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177109.496961:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7332
00000100:00000001:3.0:1786177109.496965:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1786177109.496966:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177109.496969:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000400:00000200:0.0:1786177109.496973:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177109.496974:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1786177109.496978:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177109.496984:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177109.496987:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177109.496993:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177109.496998:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177109.496999:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177109.497002:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177109.497004:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177109.497007:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:0.0:1786177109.497009:0:24919:0:(genops.c:813:class_conn2export()) Process entered
02000000:00000001:3.0:1786177109.497010:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1786177109.497013:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
02000000:00000001:3.0:1786177109.497014:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177109.497017:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00100000:3.0:1786177109.497021:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856337792:192.168.202.110@tcp:250:kworker.0
00000020:00000040:0.0:1786177109.497022:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177109.497029:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:3.0:1786177109.497030:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:0.0:1786177109.497034:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:3.0:1786177109.497035:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177109.497039:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177109.497040:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177109.497043:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1786177109.497044:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177109.497048:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000010:3.0:1786177109.497050:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:0.0:1786177109.497051:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177109.497055:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177109.497057:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177109.497062:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1786177109.497063:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177109.497068:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177109.497071:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1786177109.497078:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
02000000:00000001:0.0:1786177109.497079:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177109.497084:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177109.497087:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000010:3.0:1786177109.497088:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000008:00100000:0.0:1786177109.497093:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000200:3.0:1786177109.497097:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856337792, portal 25
00000008:00000001:0.0:1786177109.497098:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177109.497102:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.497105:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000010:0.0:1786177109.497109:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41aa00.
00000100:00000001:3.0:1786177109.497109:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761666432 : -114547947885184 : ffff97d1ba41ad80)
00000020:00000040:0.0:1786177109.497114:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177109.497119:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177109.497121:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177109.497121:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177109.497125:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177109.497128:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177109.497131:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177109.497139:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000100:00000001:3.0:1786177109.497139:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177109.497146:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177109.497147:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177109.497151:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00100000:0.0:1786177109.497157:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1786177109.497160:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856337792, offset 0
00000400:00000200:3.0:1786177109.497172:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177109.497173:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177109.497177:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177109.497182:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da0255a4] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177109.497185:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177109.497196:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1786177109.497198:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177109.497202:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:0.0:1786177109.497205:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1786177109.497210:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:0.0:1786177109.497212:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:3.0:1786177109.497216:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1786177109.497217:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177109.497221:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1786177109.497221:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177109.497228:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254810:254810:256:4294967295] 192.168.202.110@tcp LPNI seq info [254799:254799:8:4294967295]
00000100:00000001:0.0:1786177109.497229:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177109.497235:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177109.497237:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7332
00000400:00000200:3.0:1786177109.497248:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177109.497253:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177109.497257:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000010:3.0:1786177109.497261:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000400:00000200:0.0:1786177109.497264:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177109.497268:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1786177109.497272:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786177109.497277:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:3.0:1786177109.497279:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000100:00000001:0.0:1786177109.497283:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.1:1786177109.497288:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786177109.497291:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:3.1:1786177109.497293:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:3.0:1786177109.497300:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177109.497302:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00000001:0.0:1786177109.497306:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177109.497308:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:0.0:1786177109.497311:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177109.497315:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000200:3.0:1786177109.497316:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1786177109.497320:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:3.0:1786177109.497323:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000020:00000040:0.0:1786177109.497325:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000010:3.0:1786177109.497327:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000020:00000001:0.0:1786177109.497329:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:3.0:1786177109.497333:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:0.0:1786177109.497334:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000200:3.0:1786177109.497339:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177109.497342:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177109.497348:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177109.497354:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177109.497360:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177109.497364:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1786177109.497366:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000040:3.0:1786177109.497370:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177109.497374:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177109.497381:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.497383:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.497386:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:0.0:1786177109.497391:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177109.497392:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177109.497397:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177109.497398:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786177109.497403:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.497404:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302009037
00000100:00000001:0.0:1786177109.497408:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177109.497412:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000010:3.0:1786177109.497412:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37cc0.
00000008:00000001:0.0:1786177109.497418:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177109.497420:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.497422:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000010:0.0:1786177109.497426:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419500.
00000100:00000001:3.0:1786177109.497430:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302009037
00000020:00000040:0.0:1786177109.497431:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000010:3.0:1786177109.497436:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37240.
02000000:00000001:0.0:1786177109.497437:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177109.497441:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.497441:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177109.497446:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177109.497447:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177109.497451:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177109.497452:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177109.497455:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.497455:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177109.497461:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177109/real 1786177109]  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786177109.497463:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000100:00000001:0.0:1786177109.497470:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177109.497471:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177109.497474:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177109.497476:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00100000:0.0:1786177109.497479:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177109.497480:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177109.497484:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177109.497488:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1786177109.497496:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:3.0:1786177109.497496:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177109.497501:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177109.497540:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.497554:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:0.0:1786177109.497557:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b6718a9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.497558:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.497564:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177109.497567:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177109.497576:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177109.497578:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177109.497584:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177109.497588:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177109.497591:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177109.497596:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:3.0:1786177109.497596:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177109.497601:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177109.497620:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177109.497625:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7332
00000100:00000001:3.0:1786177109.497627:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000400:00000200:0.0:1786177109.497633:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177109.497633:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.497636:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177109.497639:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1786177109.497642:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177109.497643:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177109.497646:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177109.497648:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1786177109.497650:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:0.0:1786177109.497656:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00080000:3.0:1786177109.497657:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177109.497661:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1786177109.497663:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177109.497667:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177109.497670:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177109.497675:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177109.497680:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177109.497682:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177109.497686:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177109.497690:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00100000:3.0:1786177109.497692:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856337792:192.168.202.110@tcp:250:kworker.0
00000020:00000001:0.0:1786177109.497693:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177109.497699:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:3.0:1786177109.497704:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177109.497707:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177109.497709:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177109.497711:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177109.497717:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41ad80 x1872934856337792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177109.497718:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177109.497723:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177109.497729:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177109.497736:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177109.497736:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000001:0.0:1786177109.497742:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177109.497742:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000001:0.0:1786177109.497746:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:3.0:1786177109.497747:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177109.497751:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177109.497752:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177109.497756:0:2391:0:(genops.c:1238:class_import_put()) Process entered
02000000:00000001:0.0:1786177109.497757:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177109.497760:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:0.0:1786177109.497761:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000020:00000001:3.0:1786177109.497764:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000008:00100000:0.0:1786177109.497766:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000010:3.0:1786177109.497768:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
00000008:00000001:0.0:1786177109.497774:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177109.497775:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000001:0.0:1786177109.497779:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177109.497785:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a680.
02000000:00000001:3.0:1786177109.497786:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177109.497790:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41ad80.
00000020:00000040:0.0:1786177109.497791:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177109.497794:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
02000000:00000001:0.0:1786177109.497796:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177109.497797:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177109.497801:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.497802:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177109.497806:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177109.497809:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177109.497811:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.497812:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177109.497816:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.497818:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000010:0.0:1786177109.497819:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4c00.
00000100:00000001:3.0:1786177109.497820:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177109.497823:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1786177109.497826:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177109.497828:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177109.497836:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177109.497837:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177109.497840:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177109.497851:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1786177109.497851:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177109.497855:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177109.497857:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177109.497858:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177109.497866:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856337920:192.168.202.110@tcp:38:kworker.0
00000100:00000040:0.0:1786177109.497870:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000911ee1e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.497871:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177109.497874:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177109.497877:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177109.497881:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177109.497885:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177109.497889:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177109.497890:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177109.497896:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177109.497897:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1786177109.497899:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00080000:0.0:1786177109.497906:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000010:3.0:1786177109.497906:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177109.497910:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:3.0:1786177109.497911:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856337920, portal 10
00000100:00080000:0.0:1786177109.497913:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177109.497914:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177109.497917:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761665536 : -114547947886080 : ffff97d1ba41aa00)
00000100:00000001:0.0:1786177109.497920:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:3.0:1786177109.497926:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177109.497927:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7332
00000400:00000200:0.0:1786177109.497937:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177109.497941:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:0.0:1786177109.497945:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1786177109.497948:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177109.497950:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1786177109.497951:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177109.497957:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856337920, offset 0
00000100:00000040:0.0:1786177109.497959:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177109.497964:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177109.497965:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177109.497970:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177109.497972:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177109.497976:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177109.497977:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:0.0:1786177109.497980:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:3.0:1786177109.497982:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:0.0:1786177109.497986:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000400:00000200:3.0:1786177109.497987:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:0.0:1786177109.497992:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:3.0:1786177109.497992:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177109.497996:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000040:0.0:1786177109.497998:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000400:00000200:3.0:1786177109.497999:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254811:254811:256:4294967295] 192.168.202.110@tcp LPNI seq info [254800:254800:8:4294967295]
00000020:00000001:0.0:1786177109.498004:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:3.0:1786177109.498006:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:0.0:1786177109.498011:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:3.0:1786177109.498017:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1786177109.498021:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177109.498027:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1786177109.498027:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177109.498031:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000040:0.0:1786177109.498034:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177109.498038:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.0:1786177109.498039:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:0.0:1786177109.498042:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000800:00000200:3.0:1786177109.498044:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000100:00080000:0.0:1786177109.498049:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:3.1:1786177109.498053:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177109.498056:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177109.498057:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
02000000:00000001:0.0:1786177109.498061:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000100:3.0:1786177109.498065:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177109.498067:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
02000000:00000001:0.0:1786177109.498068:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177109.498070:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:0.0:1786177109.498076:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177109.498076:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1786177109.498082:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:3.0:1786177109.498082:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000008:00100000:0.0:1786177109.498086:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000010:3.0:1786177109.498086:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177109.498088:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000008:00000001:0.0:1786177109.498093:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177109.498094:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177109.498097:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177109.498104:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a300.
00000020:00000040:0.0:1786177109.498109:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177109.498111:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1786177109.498113:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177109.498116:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177109.498119:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:3.0:1786177109.498119:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177109.498123:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177109.498126:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177109.498134:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:0.0:1786177109.498141:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.498149:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177109.498151:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177109.498151:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177109.498157:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177109.498163:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000100:00000001:0.0:1786177109.498168:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177109.498169:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177109.498173:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.498174:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302005258
00000100:00000001:3.0:1786177109.498179:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302009037
00000100:00000040:0.0:1786177109.498183:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b0d3569] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498183:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302005258
00000100:00000001:3.0:1786177109.498187:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302009037
00000100:00000001:3.0:1786177109.498192:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177109.498197:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.498200:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1786177109.498206:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177109.498207:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177109/real 1786177109]  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177109.498214:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7342)
00000400:00000001:3.0:1786177109.498216:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177109.498220:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177109.498221:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177109.498224:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177109.498225:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177109.498227:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177109.498229:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177109.498235:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498240:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177109.498241:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498242:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177109.498244:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177109.498246:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498252:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177109.498254:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498260:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177109.498262:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498264:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177109.498265:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498266:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177109.498269:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177109.498271:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177109.498274:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177109.498277:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177109.498281:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177109.498288:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856337920:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177109.498292:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177109.498296:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498302:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177109.498304:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177109.498306:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177109.498308:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177109.498310:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177109.498313:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177109.498316:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177109.498318:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177109.498320:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177109.498322:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177109.498325:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177109.498327:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177109.498331:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177109.498333:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177109.498337:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177109.498340:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856338048, portal 10
00000100:00000001:3.0:1786177109.498342:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177109.498343:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761660160 : -114547947891456 : ffff97d1ba419500)
00000100:00000040:3.0:1786177109.498349:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498355:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177109.498357:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177109.498360:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177109.498362:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856338048, offset 0
00000400:00000200:3.0:1786177109.498367:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177109.498370:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177109.498374:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177109.498377:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177109.498379:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177109.498381:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177109.498383:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177109.498386:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254812:254812:256:4294967295] 192.168.202.110@tcp LPNI seq info [254801:254801:8:4294967295]
00000400:00000200:3.0:1786177109.498390:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177109.498396:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177109.498401:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177109.498404:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177109.498408:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177109.498411:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786177109.498415:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177109.498418:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:3.0:1786177109.498421:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177109.498424:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177109.498427:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177109.498430:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177109.498434:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177109.498435:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177109.498438:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177109.498442:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498450:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177109.498453:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498460:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.498462:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177109.498465:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177109.498468:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177109.498471:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177109.498474:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302005383
00000100:00000001:3.0:1786177109.498476:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302009037
00000100:00000001:3.0:1786177109.498477:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302005383
00000100:00000001:3.0:1786177109.498479:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302009037
00000100:00000001:3.0:1786177109.498481:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177109.498483:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.498484:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177109.498487:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177109/real 1786177109]  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177109.498493:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177109.498495:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177109.498497:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177109.498500:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177109.498502:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177109.498504:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177109.498508:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177109.498539:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498546:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177109.498548:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498550:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177109.498551:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177109.498554:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498559:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177109.498561:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498566:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177109.498567:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498568:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177109.498569:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498570:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177109.498571:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177109.498573:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177109.498575:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177109.498576:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177109.498579:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177109.498584:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177109.498587:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498589:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177109.498591:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41aa00 x1872934856337920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498596:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177109.498597:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177109.498598:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177109.498601:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177109.498602:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177109.498603:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177109.498604:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177109.498606:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:3.0:1786177109.498607:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177109.498621:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177109.498623:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41aa00.
00000100:00000001:3.0:1786177109.498625:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177109.498626:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498627:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177109.498630:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419500 x1872934856338048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498634:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177109.498636:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177109.498637:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177109.498639:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177109.498640:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177109.498641:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177109.498642:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177109.498643:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:3.0:1786177109.498644:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177109.498645:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177109.498645:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419500.
00000100:00000001:3.0:1786177109.498647:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177109.498648:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498649:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498651:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177109.498652:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.498653:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177109.498655:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177109.498656:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177109.498658:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498663:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177109.498664:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177109.498666:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177109.498667:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177109.498668:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177109.498671:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177109.498674:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177109.498676:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177109.498678:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177109.498679:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177109.498683:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177109.498685:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177109.498689:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177109.498692:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177109.498696:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177109.498700:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856338176, portal 4
00000100:00000001:3.0:1786177109.498702:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177109.498704:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761664640 : -114547947886976 : ffff97d1ba41a680)
00000100:00000040:3.0:1786177109.498711:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498719:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177109.498721:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177109.498724:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177109.498726:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856338176, offset 0
00000400:00000200:3.0:1786177109.498731:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177109.498734:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177109.498737:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177109.498740:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177109.498742:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177109.498744:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177109.498746:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177109.498748:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254813:254813:256:4294967295] 192.168.202.110@tcp LPNI seq info [254802:254802:8:4294967295]
00000400:00000200:3.0:1786177109.498751:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177109.498756:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177109.498758:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177109.498761:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177109.498763:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177109.498765:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786177109.498768:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177109.498770:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:3.0:1786177109.498773:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177109.498774:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177109.498776:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177109.498779:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177109.498781:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177109.498782:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177109.498784:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177109.498787:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498793:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177109.498795:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498799:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.498800:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177109.498803:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177109.498805:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177109.498808:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177109.498809:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302005259
00000100:00000001:3.0:1786177109.498811:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302009037
00000100:00000001:3.0:1786177109.498813:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302005259
00000100:00000001:3.0:1786177109.498816:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302009037
00000100:00000001:3.0:1786177109.498818:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177109.498819:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.498821:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177109.498823:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177109/real 1786177109]  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177109.498828:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177109.498829:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177109.498830:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177109.498832:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177109.498833:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177109.498834:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177109.498836:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177109.498842:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498846:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177109.498847:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498848:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177109.498849:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177109.498851:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498855:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177109.498857:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498861:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177109.498863:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498864:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177109.498865:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.498866:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177109.498868:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177109.498870:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177109.498872:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177109.498874:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177109.498876:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177109.498881:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177109.498885:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177109.498887:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498890:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177109.498891:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177109.498893:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177109.498894:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177109.498895:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177109.498897:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177109.498899:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177109.498900:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177109.498901:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177109.498902:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177109.498904:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177109.498905:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177109.498907:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177109.498909:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177109.498911:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177109.498913:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856338304, portal 4
00000100:00000001:3.0:1786177109.498915:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177109.498916:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761663744 : -114547947887872 : ffff97d1ba41a300)
00000100:00000040:3.0:1786177109.498919:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498924:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177109.498926:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177109.498927:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177109.498929:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856338304, offset 0
00000400:00000200:3.0:1786177109.498931:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177109.498933:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177109.498935:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177109.498937:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177109.498938:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177109.498940:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177109.498941:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177109.498943:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254814:254814:256:4294967295] 192.168.202.110@tcp LPNI seq info [254803:254803:8:4294967295]
00000400:00000200:3.0:1786177109.498946:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177109.498951:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177109.498954:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177109.498956:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177109.498958:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177109.498960:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1786177109.498963:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177109.498965:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:3.0:1786177109.498967:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177109.498969:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177109.498970:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177109.498973:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177109.498975:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177109.498976:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177109.498977:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177109.498980:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498985:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177109.498987:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.498996:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.498997:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177109.498999:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177109.499001:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177109.499003:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177109.499004:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302005267
00000100:00000001:3.0:1786177109.499006:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302009037
00000100:00000001:3.0:1786177109.499008:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302005267
00000100:00000001:3.0:1786177109.499009:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302009037
00000100:00000001:3.0:1786177109.499010:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177109.499012:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.499013:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177109.499015:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177109/real 1786177109]  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177109.499019:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177109.499020:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177109.499021:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177109.499022:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177109.499023:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177109.499025:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177109.499027:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177109.499032:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.499036:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177109.499037:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.499038:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177109.499038:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177109.499041:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.499046:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177109.499048:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.499053:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177109.499054:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.499055:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177109.499056:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.499057:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177109.499058:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177109.499060:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177109.499061:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177109.499063:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177109.499065:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177109.499069:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177109.499072:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.499073:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177109.499075:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a680 x1872934856338176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.499079:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177109.499080:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177109.499081:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177109.499083:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177109.499084:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177109.499085:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177109.499087:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177109.499088:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:3.0:1786177109.499089:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177109.499091:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177109.499092:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a680.
00000100:00000001:3.0:1786177109.499094:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177109.499095:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.499097:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177109.499100:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a300 x1872934856338304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177109.499108:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177109.499109:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177109.499111:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177109.499113:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177109.499115:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177109.499116:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177109.499118:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177109.499120:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:3.0:1786177109.499123:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177109.499125:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177109.499126:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a300.
00000100:00000001:3.0:1786177109.499129:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177109.499130:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.499132:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177109.499135:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177109.499137:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177109.499139:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177109.499141:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177110.072624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177110.072633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177110.072641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177110.072642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177110.072645:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177110.072647:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177110.072742:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177110.072749:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177110.072754:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177110.520723:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177110.520733:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177110.520736:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177110.520745:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177110.520751:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177110.520757:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177110.520761:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177110.520764:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177110.520768:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177111.096698:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177111.096709:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177111.096712:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177111.096716:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177111.096722:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177111.096725:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177111.096727:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177111.096730:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177111.096735:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177111.096736:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177111.096738:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177111.096740:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177111.096741:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177111.096743:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177111.096745:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177111.096746:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177111.096750:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177111.096752:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177111.096846:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177111.096853:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177111.096857:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177111.544622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177111.544629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177111.544631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177111.544636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177111.544639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177111.544642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177111.544645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177111.544647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177111.544648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177112.120635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177112.120643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177112.120647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177112.120648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177112.120651:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177112.120652:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177112.120719:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177112.120727:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177112.120733:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177112.568633:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177112.568641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177112.568644:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177112.568650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177112.568653:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177112.568659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177112.568662:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177112.568665:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177112.568668:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177113.144745:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177113.144769:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177113.144776:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177113.144798:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177113.144802:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177113.144805:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177113.144807:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177113.144819:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177113.144822:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177113.144824:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177113.144825:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177113.144827:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177113.144892:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177113.144896:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177113.144899:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177113.336647:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302009037, at 4302009088
00000100:00000010:0.0:1786177113.336662:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37cc0.
00000100:00000040:1.0:1786177113.336663:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302009037, at 4302009088
00000100:00000010:1.0:1786177113.336690:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37240.
00000100:00000001:3.0:1786177113.592707:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177113.592717:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177113.592720:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177113.592726:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177113.592731:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177113.592735:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177113.592738:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177113.592742:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177113.592745:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177114.168688:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177114.168697:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177114.168705:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177114.168706:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177114.168709:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177114.168711:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177114.168796:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177114.168801:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177114.168806:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786177114.616640:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177114.616657:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1786177114.616658:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177114.616663:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177114.616664:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177114.616666:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177114.616672:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:3.0:1786177114.616673:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177114.616677:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177114.616683:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177114.616683:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786177114.616688:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177114.616691:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.616694:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177114.616697:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7337
00000400:00000200:3.0:1786177114.616711:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177114.616721:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177114.616726:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177114.616729:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.616732:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177114.616734:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.616737:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177114.616741:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177114.616743:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177114.616746:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177114.616749:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177114.616751:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177114.616753:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177114.616755:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.616756:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177114.616758:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.616759:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177114.616762:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177114.616765:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177114.616766:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177114.616769:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177114.616772:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177114.616774:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177114.616776:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177114.616778:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177114.616788:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69dc00.
00000020:00000040:3.0:1786177114.616791:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177114.616795:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177114.616796:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177114.616799:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177114.616802:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177114.616807:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177114.616815:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:3.0:1786177114.616823:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177114.616832:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177114.616841:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177114.616845:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177114.616850:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097fe80e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177114.616872:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177114.616875:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177114.616878:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177114.616879:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177114.616880:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177114.616882:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177114.616884:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7337
00000400:00000200:3.0:1786177114.616887:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177114.616889:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177114.616891:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177114.616894:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.616896:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177114.616898:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.616900:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177114.616902:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177114.616903:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:0.0:1786177114.616904:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1786177114.616904:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786177114.616909:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1786177114.616909:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786177114.616911:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786177114.616912:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1786177114.616913:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177114.616917:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.616920:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177114.616921:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177114.616924:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.616928:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177114.616933:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786177114.616935:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177114.616937:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177114.616938:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177114.616944:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177114.616946:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1786177114.616947:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.616950:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177114.616951:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177114.616956:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1786177114.616959:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:3.0:1786177114.616961:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177114.616965:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177114.616967:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:3.0:1786177114.616971:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e680.
02000000:00000001:0.0:1786177114.616973:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177114.616977:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00100000:0.0:1786177114.616982:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856338432:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1786177114.616984:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177114.616987:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.616989:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1786177114.616992:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177114.616995:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177114.616996:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177114.616999:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177114.617000:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177114.617003:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177114.617008:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000010:0.0:1786177114.617010:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177114.617014:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177114.617015:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177114.617021:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1786177114.617024:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1786177114.617026:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000100:00000001:3.0:1786177114.617035:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000010:0.0:1786177114.617036:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000001:3.0:1786177114.617037:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177114.617042:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1839131] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177114.617046:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856338432, portal 25
00000100:00000001:3.0:1786177114.617051:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.617052:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177114.617054:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764299776 : -114547945251840 : ffff97d1ba69dc00)
00000100:00080000:3.0:1786177114.617056:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177114.617060:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177114.617063:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:0.0:1786177114.617064:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786177114.617065:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177114.617070:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177114.617076:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7337
00000100:00000001:0.0:1786177114.617079:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:3.0:1786177114.617082:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177114.617086:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177114.617089:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000010:0.0:1786177114.617090:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00080000:3.0:1786177114.617095:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:0.0:1786177114.617096:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856338432, offset 0
00000100:00000040:3.0:1786177114.617102:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617105:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786177114.617107:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786177114.617109:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617112:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177114.617113:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177114.617115:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177114.617116:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177114.617117:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:0.0:1786177114.617119:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1786177114.617120:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177114.617124:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177114.617126:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177114.617129:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1786177114.617130:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177114.617135:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617137:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177114.617139:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:0.0:1786177114.617140:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1786177114.617143:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617146:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177114.617149:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:3.0:1786177114.617149:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177114.617153:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1786177114.617156:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1786177114.617158:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177114.617160:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617161:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177114.617165:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1786177114.617167:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254815:254815:256:4294967295] 192.168.202.110@tcp LPNI seq info [254804:254804:8:4294967295]
00000008:00000001:3.0:1786177114.617168:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177114.617172:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177114.617176:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:3.0:1786177114.617177:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ed80.
00000020:00000040:3.0:1786177114.617181:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177114.617185:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177114.617188:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177114.617189:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786177114.617190:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177114.617193:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177114.617196:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177114.617200:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000010:3.0:1786177114.617201:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000001:3.0:1786177114.617205:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1786177114.617208:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000100:00100000:3.0:1786177114.617210:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177114.617218:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177114.617219:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177114.617221:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ccd6c6d2] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177114.617227:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177114.617229:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.0:1786177114.617232:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786177114.617232:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177114.617234:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177114.617236:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177114.617240:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:0.0:1786177114.617241:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00080000:3.0:1786177114.617243:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7337
00000400:00000200:3.0:1786177114.617249:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177114.617254:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177114.617257:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:0.1:1786177114.617257:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:3.0:1786177114.617262:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617265:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1786177114.617265:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:3.0:1786177114.617269:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617272:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177114.617275:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000100:0.0:1786177114.617277:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000020:00000040:3.0:1786177114.617278:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177114.617281:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000010:0.0:1786177114.617284:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000020:00000040:3.0:1786177114.617284:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177114.617287:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177114.617290:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177114.617292:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617294:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786177114.617295:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1786177114.617296:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617302:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177114.617303:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177114.617306:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617307:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177114.617308:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177114.617309:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177114.617312:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177114.617315:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1786177114.617315:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177114.617318:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177114.617321:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177114.617323:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177114.617325:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c380.
00000400:00000200:0.0:1786177114.617326:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000020:00000040:3.0:1786177114.617327:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000400:00000010:0.0:1786177114.617329:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
02000000:00000001:3.0:1786177114.617330:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177114.617333:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177114.617335:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786177114.617336:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:3.0:1786177114.617336:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177114.617337:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177114.617341:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000200:0.0:1786177114.617343:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177114.617345:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177114.617353:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177114.617358:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786177114.617359:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177114.617362:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177114.617368:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177114.617368:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000833bf1c9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617380:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.617386:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177114.617387:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177114.617390:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177114.617393:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177114.617394:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177114.617396:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177114.617396:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177114.617401:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177114.617405:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7337
00000400:00000200:0.0:1786177114.617407:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177114.617411:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177114.617414:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302014157
00000400:00000200:3.0:1786177114.617415:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177114.617419:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177114.617422:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617424:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:0.0:1786177114.617426:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000040:3.0:1786177114.617426:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617430:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177114.617432:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177114.617433:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177114.617434:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177114.617436:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177114.617437:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:0.0:1786177114.617439:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000001:3.0:1786177114.617439:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177114.617444:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617446:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177114.617447:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617449:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177114.617450:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:0.0:1786177114.617451:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302014157
00000100:00080000:3.0:1786177114.617453:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177114.617457:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177114.617459:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1851344e0.
02000000:00000001:3.0:1786177114.617459:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177114.617461:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177114.617465:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.617468:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177114.617468:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177114.617471:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177114.617474:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.617475:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177114.617475:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177114.617482:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d880.
00000100:00000001:0.0:1786177114.617483:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177114.617485:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:0.0:1786177114.617488:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000001:3.0:1786177114.617489:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177114.617492:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177114.617494:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177114.617496:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:0.0:1786177114.617498:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177114/real 1786177114]  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177114.617499:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177114.617503:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000400:00000001:0.0:1786177114.617533:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177114.617535:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177114.617537:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177114.617540:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000100:00000001:3.0:1786177114.617540:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177114.617543:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177114.617547:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177114.617552:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177114.617552:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177114.617594:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:0.0:1786177114.617600:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177114.617600:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177114.617609:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097f9b8db] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617615:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177114.617620:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177114.617624:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.617625:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177114.617630:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177114.617631:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7348)
00000100:00100000:0.0:1786177114.617636:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617643:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177114.617650:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617658:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177114.617662:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.617664:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177114.617665:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.617668:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177114.617671:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177114.617674:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177114.617679:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177114.617682:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177114.617688:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177114.617697:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856338432:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177114.617701:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.617703:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177114.617706:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69dc00 x1872934856338432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617711:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177114.617714:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177114.617716:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177114.617719:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177114.617721:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177114.617723:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177114.617725:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177114.617727:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177114.617731:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177114.617733:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177114.617734:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69dc00.
00000100:00000001:0.0:1786177114.617736:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177114.617737:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.617740:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.617747:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177114.617748:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.617749:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177114.617750:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177114.617752:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177114.617754:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617759:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177114.617761:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177114.617762:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177114.617764:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177114.617765:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177114.617767:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177114.617770:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177114.617771:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177114.617772:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177114.617774:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177114.617776:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177114.617777:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177114.617780:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177114.617782:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177114.617785:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177114.617788:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856338560, portal 10
00000100:00000001:0.0:1786177114.617790:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177114.617791:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00000040:0.0:1786177114.617796:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617801:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177114.617803:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177114.617804:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177114.617806:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856338560, offset 0
00000400:00000200:0.0:1786177114.617808:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177114.617811:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177114.617814:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177114.617816:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177114.617818:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177114.617820:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177114.617822:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177114.617824:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254816:254816:256:4294967295] 192.168.202.110@tcp LPNI seq info [254805:254805:8:4294967295]
00000400:00000200:0.0:1786177114.617827:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177114.617832:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177114.617835:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177114.617837:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177114.617840:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177114.617842:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786177114.617846:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177114.617847:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786177114.617850:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177114.617852:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177114.617854:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177114.617856:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177114.617858:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177114.617859:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177114.617861:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177114.617863:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617874:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177114.617876:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617880:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.617881:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177114.617883:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177114.617887:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177114.617889:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177114.617890:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302010378
00000100:00000001:0.0:1786177114.617892:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302014157
00000100:00000001:0.0:1786177114.617894:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302010378
00000100:00000001:0.0:1786177114.617896:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302014157
00000100:00000001:0.0:1786177114.617897:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177114.617898:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.617900:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177114.617902:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177114/real 1786177114]  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177114.617907:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177114.617908:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177114.617910:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177114.617911:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177114.617912:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177114.617914:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177114.617916:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177114.617921:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617925:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177114.617927:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.617928:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177114.617929:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177114.617931:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617938:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177114.617942:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617949:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177114.617952:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.617954:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177114.617955:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.617957:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177114.617959:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177114.617962:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177114.617966:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177114.617968:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177114.617973:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177114.617982:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177114.617986:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177114.617990:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.617999:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177114.618000:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177114.618002:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177114.618004:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177114.618006:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177114.618009:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177114.618011:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177114.618013:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177114.618014:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177114.618016:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177114.618018:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177114.618020:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177114.618022:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177114.618024:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177114.618026:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177114.618028:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856338688, portal 10
00000100:00000001:0.0:1786177114.618029:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177114.618031:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:0.0:1786177114.618035:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618040:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177114.618041:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177114.618043:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177114.618044:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856338688, offset 0
00000400:00000200:0.0:1786177114.618047:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177114.618049:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177114.618051:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177114.618054:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177114.618055:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177114.618057:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177114.618059:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177114.618061:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254817:254817:256:4294967295] 192.168.202.110@tcp LPNI seq info [254806:254806:8:4294967295]
00000400:00000200:0.0:1786177114.618064:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177114.618068:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177114.618072:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177114.618074:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177114.618077:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177114.618079:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786177114.618082:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177114.618083:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786177114.618086:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177114.618088:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177114.618090:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177114.618092:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177114.618094:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177114.618095:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177114.618097:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177114.618099:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618105:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177114.618107:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618111:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.618112:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177114.618114:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177114.618116:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177114.618118:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177114.618119:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302010503
00000100:00000001:0.0:1786177114.618121:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302014157
00000100:00000001:0.0:1786177114.618122:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302010503
00000100:00000001:0.0:1786177114.618124:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302014157
00000100:00000001:0.0:1786177114.618125:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177114.618127:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.618128:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177114.618130:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177114/real 1786177114]  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177114.618135:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177114.618137:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177114.618138:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177114.618140:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177114.618141:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177114.618142:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177114.618144:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177114.618149:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618153:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177114.618154:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618155:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177114.618156:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177114.618158:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618162:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177114.618164:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618168:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177114.618170:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618171:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177114.618172:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618173:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177114.618175:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177114.618176:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177114.618178:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177114.618180:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177114.618182:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177114.618192:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177114.618195:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177114.618197:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618200:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177114.618201:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177114.618203:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177114.618204:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177114.618205:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177114.618207:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177114.618208:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177114.618209:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177114.618210:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177114.618211:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177114.618213:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7800.
02000000:00000001:0.0:1786177114.618214:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177114.618217:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177114.618218:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177114.618221:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177114.618222:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856338816, portal 4
00000100:00000001:0.0:1786177114.618224:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177114.618225:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764293504 : -114547945258112 : ffff97d1ba69c380)
00000100:00000040:0.0:1786177114.618230:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618234:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177114.618236:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177114.618237:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177114.618239:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856338816, offset 0
00000400:00000200:0.0:1786177114.618241:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177114.618244:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177114.618246:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177114.618248:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177114.618250:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177114.618251:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177114.618253:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177114.618255:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254818:254818:256:4294967295] 192.168.202.110@tcp LPNI seq info [254807:254807:8:4294967295]
00000400:00000200:0.0:1786177114.618258:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177114.618262:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177114.618265:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177114.618267:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177114.618269:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177114.618270:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786177114.618273:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177114.618275:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786177114.618278:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177114.618279:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177114.618281:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177114.618283:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177114.618285:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177114.618287:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177114.618288:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177114.618290:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618296:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177114.618298:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618302:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.618303:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177114.618305:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177114.618306:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177114.618308:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177114.618310:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302010378
00000100:00000001:0.0:1786177114.618312:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302014157
00000100:00000001:0.0:1786177114.618313:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302010378
00000100:00000001:0.0:1786177114.618315:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302014157
00000100:00000001:0.0:1786177114.618316:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177114.618317:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.618318:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177114.618320:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177114/real 1786177114]  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177114.618325:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177114.618326:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177114.618326:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177114.618328:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177114.618329:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177114.618330:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177114.618332:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177114.618336:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618341:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177114.618342:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618343:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177114.618344:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177114.618346:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618350:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177114.618353:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618356:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177114.618359:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618360:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177114.618361:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618362:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177114.618363:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177114.618365:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177114.618367:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177114.618368:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177114.618370:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177114.618375:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177114.618377:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177114.618379:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618383:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177114.618383:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177114.618385:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177114.618386:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177114.618387:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177114.618388:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177114.618390:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177114.618391:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177114.618392:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177114.618393:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177114.618396:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177114.618397:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177114.618399:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177114.618401:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177114.618402:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177114.618404:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856338944, portal 4
00000100:00000001:0.0:1786177114.618406:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177114.618407:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
00000100:00000040:0.0:1786177114.618410:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618415:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177114.618416:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177114.618417:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177114.618419:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856338944, offset 0
00000400:00000200:0.0:1786177114.618421:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177114.618423:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177114.618425:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177114.618426:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177114.618428:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177114.618429:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177114.618430:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177114.618433:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254819:254819:256:4294967295] 192.168.202.110@tcp LPNI seq info [254808:254808:8:4294967295]
00000400:00000200:0.0:1786177114.618435:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177114.618439:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177114.618442:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177114.618444:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177114.618446:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177114.618447:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1786177114.618450:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177114.618451:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1786177114.618454:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177114.618455:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177114.618457:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177114.618459:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177114.618461:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177114.618462:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177114.618464:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177114.618466:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618475:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177114.618477:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618481:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.618482:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177114.618483:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177114.618485:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177114.618487:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177114.618488:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302010386
00000100:00000001:0.0:1786177114.618489:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302014157
00000100:00000001:0.0:1786177114.618491:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302010386
00000100:00000001:0.0:1786177114.618492:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302014157
00000100:00000001:0.0:1786177114.618494:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177114.618496:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.618497:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177114.618499:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177114/real 1786177114]  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177114.618507:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177114.618527:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177114.618530:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177114.618533:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177114.618535:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177114.618538:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177114.618542:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177114.618548:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618553:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177114.618554:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618556:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177114.618557:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177114.618575:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618581:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177114.618584:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618588:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177114.618590:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618591:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177114.618592:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618593:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177114.618595:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177114.618596:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177114.618598:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177114.618599:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177114.618601:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177114.618607:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856338944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177114.618610:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618611:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177114.618613:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856338560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618617:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177114.618618:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177114.618620:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177114.618622:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177114.618623:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177114.618625:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177114.618626:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177114.618628:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786177114.618630:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177114.618631:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177114.618632:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:0.0:1786177114.618634:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177114.618635:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618636:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177114.618638:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856338688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618643:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177114.618643:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177114.618644:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177114.618646:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177114.618647:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177114.618648:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177114.618650:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177114.618652:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786177114.618654:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177114.618655:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177114.618657:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:0.0:1786177114.618659:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177114.618661:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618663:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177114.618666:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c380 x1872934856338816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618674:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177114.618675:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177114.618677:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7800.
02000000:00000001:0.0:1786177114.618679:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177114.618681:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177114.618683:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177114.618685:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177114.618686:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786177114.618689:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177114.618691:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177114.618693:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c380.
00000100:00000001:0.0:1786177114.618696:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177114.618697:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618699:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177114.618702:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856338944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177114.618707:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177114.618708:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177114.618709:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177114.618710:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177114.618711:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177114.618712:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177114.618713:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177114.618714:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786177114.618715:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177114.618716:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177114.618717:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:0.0:1786177114.618718:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177114.618719:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618720:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177114.618722:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177114.618723:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177114.618724:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177114.618726:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177115.192704:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177115.192712:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177115.192717:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177115.192718:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177115.192720:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177115.192721:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177115.192806:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177115.192813:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177115.192816:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177115.640668:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177115.640674:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177115.640676:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177115.640681:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177115.640684:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177115.640687:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177115.640689:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177115.640691:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177115.640693:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177116.216636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177116.216641:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177116.216645:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177116.216646:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177116.216649:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177116.216650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177116.216739:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177116.216747:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177116.216752:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177116.664690:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177116.664696:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177116.664697:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177116.664701:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177116.664704:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177116.664707:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177116.664708:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177116.664710:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177116.664712:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177117.240627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177117.240633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177117.240637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177117.240638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177117.240641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177117.240642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177117.240714:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177117.240719:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177117.240722:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177117.688655:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177117.688665:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177117.688667:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177117.688672:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177117.688676:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177117.688680:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177117.688683:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177117.688686:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177117.688689:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177118.264625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177118.264630:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177118.264634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177118.264635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177118.264637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177118.264638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177118.264696:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177118.264700:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177118.264702:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177118.456607:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302014157, at 4302014208
00000100:00000040:0.0:1786177118.456615:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302014157, at 4302014208
00000100:00000010:0.0:1786177118.456630:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000010:1.0:1786177118.456632:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1851344e0.
00000100:00000001:0.0:1786177118.712616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177118.712625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177118.712627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177118.712633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177118.712636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177118.712639:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177118.712641:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177118.712644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177118.712646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177119.288632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177119.288638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177119.288642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177119.288650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177119.288652:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177119.288653:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177119.288713:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177119.288722:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177119.288726:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177119.736617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1786177119.736618:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177119.736624:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:2.0:1786177119.736627:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177119.736628:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177119.736632:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1786177119.736634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786177119.736637:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177119.736639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177119.736644:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177119.736645:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177119.736649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1786177119.736651:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7343
00000400:00000200:2.0:1786177119.736660:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177119.736660:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177119.736665:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177119.736667:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177119.736670:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177119.736672:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.736675:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177119.736677:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.736679:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177119.736681:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177119.736683:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177119.736685:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177119.736687:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177119.736689:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177119.736690:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177119.736692:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.736693:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177119.736694:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.736695:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177119.736697:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177119.736699:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177119.736701:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177119.736704:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177119.736706:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177119.736708:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177119.736709:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177119.736711:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177119.736718:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786177119.736720:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177119.736723:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177119.736725:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177119.736727:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177119.736729:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177119.736732:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177119.736737:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786177119.736742:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177119.736748:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177119.736754:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177119.736757:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177119.736760:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177119.736774:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177119.736777:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177119.736778:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177119.736780:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177119.736781:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177119.736783:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177119.736784:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7343
00000400:00000200:2.0:1786177119.736787:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177119.736789:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177119.736790:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177119.736793:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.736795:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177119.736796:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1786177119.736797:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177119.736800:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177119.736800:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177119.736802:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:0.0:1786177119.736803:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1786177119.736804:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177119.736807:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177119.736808:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177119.736810:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177119.736811:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177119.736812:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177119.736818:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.736822:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177119.736825:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.736829:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177119.736833:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177119.736837:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177119.736838:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177119.736841:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177119.736841:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177119.736845:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177119.736846:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177119.736849:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177119.736850:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177119.736851:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177119.736854:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177119.736857:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856339072:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177119.736858:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177119.736862:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:0.0:1786177119.736863:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1786177119.736866:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177119.736869:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1786177119.736869:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177119.736871:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
02000000:00000001:0.0:1786177119.736873:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786177119.736874:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177119.736877:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177119.736878:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1786177119.736879:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177119.736882:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177119.736883:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177119.736884:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177119.736887:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5800.
02000000:00000010:2.0:1786177119.736887:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
02000000:00000001:0.0:1786177119.736892:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177119.736893:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177119.736900:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1786177119.736905:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177119.736910:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000010:0.0:1786177119.736912:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000100:00000001:2.0:1786177119.736913:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177119.736919:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1786177119.736921:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000001:2.0:1786177119.736929:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786177119.736931:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856339072, portal 25
00000100:00080000:2.0:1786177119.736931:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177119.736934:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177119.736935:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177119.736936:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177119.736937:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177119.736939:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7343
00000400:00000200:2.0:1786177119.736941:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177119.736942:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177119.736943:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177119.736945:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177119.736949:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177119.736949:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177119.736951:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177119.736953:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000001:2.0:1786177119.736953:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177119.736959:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177119.736960:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177119.736961:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177119.736963:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177119.736964:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177119.736965:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177119.736966:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786177119.736969:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.736972:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177119.736974:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.736978:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177119.736981:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000001:0.0:1786177119.736983:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1786177119.736985:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:0.0:1786177119.736989:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177119.736990:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177119.736993:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:0.0:1786177119.736994:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
02000000:00000001:2.0:1786177119.736995:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:0.0:1786177119.736999:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856339072, offset 0
02000000:00000001:2.0:1786177119.737000:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177119.737004:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:0.0:1786177119.737007:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000008:00100000:2.0:1786177119.737008:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177119.737012:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177119.737013:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177119.737016:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:2.0:1786177119.737018:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000020:00000040:2.0:1786177119.737022:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:0.0:1786177119.737023:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177119.737029:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:2.0:1786177119.737029:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177119.737031:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177119.737034:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177119.737037:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177119.737039:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.737042:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1786177119.737043:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254820:254820:256:4294967295] 192.168.202.110@tcp LPNI seq info [254809:254809:8:4294967295]
02000000:00000001:2.0:1786177119.737046:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1786177119.737048:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177119.737049:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177119.737053:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000400:00000200:0.0:1786177119.737059:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786177119.737059:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177119.737066:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1786177119.737067:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177119.737074:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000100:00000001:2.0:1786177119.737075:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177119.737077:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177119.737080:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1786177119.737081:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1786177119.737085:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1786177119.737093:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786177119.737095:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786177119.737097:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:0.1:1786177119.737100:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00080000:2.0:1786177119.737106:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177119.737108:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177119.737109:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000100:0.0:1786177119.737112:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177119.737112:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:0.0:1786177119.737117:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000100:00080000:2.0:1786177119.737117:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7343
00000400:00000200:2.0:1786177119.737123:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177119.737125:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177119.737127:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177119.737130:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786177119.737132:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:2.0:1786177119.737135:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.737137:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786177119.737138:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000100:00000040:2.0:1786177119.737140:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:0.0:1786177119.737141:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:2.0:1786177119.737142:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177119.737146:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:0.0:1786177119.737147:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:2.0:1786177119.737147:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177119.737150:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000200:0.0:1786177119.737153:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177119.737153:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177119.737156:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177119.737159:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786177119.737164:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.737169:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177119.737171:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177119.737171:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.737174:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:0.0:1786177119.737177:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177119.737178:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177119.737183:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177119.737188:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.737189:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177119.737190:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177119.737193:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1786177119.737194:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177119.737197:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177119.737197:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177119.737198:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177119.737202:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:0.0:1786177119.737203:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:2.0:1786177119.737205:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.737207:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302019277
00000100:00000001:2.0:1786177119.737208:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177119.737214:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000100:00000010:0.0:1786177119.737215:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000020:00000040:2.0:1786177119.737216:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177119.737219:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177119.737220:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177119.737221:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786177119.737222:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:2.0:1786177119.737222:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177119.737223:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.737228:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302019277
02000000:00000010:2.0:1786177119.737228:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:2.0:1786177119.737233:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177119.737234:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134ae0.
00000100:00000001:0.0:1786177119.737238:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00100000:2.0:1786177119.737238:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177119.737241:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177119.737246:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.737250:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1786177119.737250:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177119.737253:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177119.737256:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177119/real 1786177119]  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177119.737258:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177119.737280:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786177119.737281:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1786177119.737283:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:2.0:1786177119.737284:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1786177119.737286:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000100:00080000:2.0:1786177119.737287:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:0.0:1786177119.737289:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000100:00000001:2.0:1786177119.737290:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:0.0:1786177119.737293:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00080000:2.0:1786177119.737293:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177119.737297:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1786177119.737297:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:0.0:1786177119.737303:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786177119.737303:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7343
00000400:00000200:2.0:1786177119.737307:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177119.737312:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177119.737316:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:0.0:1786177119.737321:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177119.737321:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.737325:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177119.737329:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.737333:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177119.737334:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000001:2.0:1786177119.737336:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:0.0:1786177119.737337:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786177119.737338:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:0.0:1786177119.737339:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000001:2.0:1786177119.737340:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786177119.737342:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786177119.737343:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786177119.737346:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00100000:0.0:1786177119.737347:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177119.737348:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177119.737353:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.737356:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177119.737359:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177119.737363:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177119.737366:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786177119.737367:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786177119.737371:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177119.737373:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177119.737375:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177119.737377:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1786177119.737378:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177119.737379:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177119.737382:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177119.737384:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177119.737389:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177119.737392:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.737397:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000010:2.0:1786177119.737397:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000020:00000040:2.0:1786177119.737400:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:0.0:1786177119.737404:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786177119.737404:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177119.737406:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.737408:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177119.737409:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786177119.737412:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786177119.737413:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177119.737415:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.737417:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000010:2.0:1786177119.737418:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:0.0:1786177119.737420:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177119.737424:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:2.0:1786177119.737428:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177119.737430:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00100000:2.0:1786177119.737434:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177119.737436:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1786177119.737445:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:0.0:1786177119.737446:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177119.737448:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177119.737453:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177119.737465:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177119.737466:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856339072:192.168.202.110@tcp:250:kworker.0
00000100:00000040:2.0:1786177119.737469:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7353)
00000100:00000001:0.0:1786177119.737474:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.737477:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177119.737481:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856339072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.737487:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177119.737490:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177119.737493:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177119.737496:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177119.737497:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177119.737499:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177119.737500:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177119.737503:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:0.0:1786177119.737505:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177119.737507:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177119.737508:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:0.0:1786177119.737552:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177119.737554:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.737556:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.737562:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177119.737564:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.737566:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177119.737567:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177119.737568:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177119.737572:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.737578:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177119.737579:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177119.737581:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177119.737582:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177119.737585:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177119.737587:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856339200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177119.737590:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177119.737592:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177119.737593:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177119.737595:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177119.737597:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177119.737599:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177119.737602:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177119.737605:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177119.737607:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177119.737610:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856339200, portal 10
00000100:00000001:0.0:1786177119.737612:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177119.737613:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178944 : -114548255372672 : ffff97d1a7edca80)
00000100:00000040:0.0:1786177119.737619:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.737624:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177119.737626:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177119.737627:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177119.737629:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856339200, offset 0
00000400:00000200:0.0:1786177119.737633:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177119.737637:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177119.737640:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177119.737643:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177119.737645:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177119.737647:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177119.737649:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177119.737652:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254821:254821:256:4294967295] 192.168.202.110@tcp LPNI seq info [254810:254810:8:4294967295]
00000400:00000200:0.0:1786177119.737656:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177119.737661:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177119.737665:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177119.737669:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177119.737672:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177119.737675:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786177119.737679:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177119.737681:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786177119.737685:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177119.737686:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177119.737689:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177119.737693:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177119.737695:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177119.737697:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177119.737699:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177119.737702:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.737710:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177119.737713:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.737720:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.737722:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177119.737725:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177119.737729:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177119.737732:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177119.737734:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302015498
00000100:00000001:0.0:1786177119.737736:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302019277
00000100:00000001:0.0:1786177119.737738:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302015498
00000100:00000001:0.0:1786177119.737740:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302019277
00000100:00000001:0.0:1786177119.737742:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177119.737744:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.737746:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177119.737750:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177119/real 1786177119]  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177119.737764:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177119.737765:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177119.737767:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177119.737770:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177119.737771:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177119.737773:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177119.737777:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177119.737786:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.737792:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177119.737794:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.737795:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177119.737797:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177119.737801:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.737808:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177119.737812:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.737819:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177119.737821:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.737823:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177119.737824:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.737826:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177119.737828:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177119.737831:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177119.737834:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177119.737837:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177119.737842:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177119.737850:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856339200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177119.737855:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177119.737858:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.737864:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177119.737866:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177119.737867:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177119.737869:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177119.737871:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177119.737874:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856339328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177119.737877:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177119.737879:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177119.737881:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177119.737882:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177119.737886:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177119.737888:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177119.737891:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177119.737893:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177119.737896:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177119.737899:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856339328, portal 10
00000100:00000001:0.0:1786177119.737900:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177119.737902:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000100:00000040:0.0:1786177119.737907:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.737914:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177119.737916:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177119.737919:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177119.737921:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856339328, offset 0
00000400:00000200:0.0:1786177119.737925:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177119.737929:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177119.737932:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177119.737935:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177119.737937:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177119.737940:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177119.737942:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177119.737945:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254822:254822:256:4294967295] 192.168.202.110@tcp LPNI seq info [254811:254811:8:4294967295]
00000400:00000200:0.0:1786177119.737949:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177119.737955:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177119.737960:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177119.737963:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177119.737966:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177119.737969:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786177119.737972:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177119.737975:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786177119.737979:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177119.737980:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177119.737983:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177119.737987:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177119.737990:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177119.737992:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177119.737995:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177119.737998:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738006:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177119.738009:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738015:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.738017:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177119.738019:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177119.738022:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177119.738025:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177119.738027:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302015623
00000100:00000001:0.0:1786177119.738030:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302019277
00000100:00000001:0.0:1786177119.738032:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302015623
00000100:00000001:0.0:1786177119.738034:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302019277
00000100:00000001:0.0:1786177119.738035:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177119.738037:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.738039:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177119.738043:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177119/real 1786177119]  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177119.738050:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177119.738051:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177119.738053:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177119.738055:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177119.738056:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177119.738058:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177119.738061:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177119.738069:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738074:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177119.738075:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738077:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177119.738079:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177119.738082:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738089:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177119.738093:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738099:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177119.738101:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738103:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177119.738104:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738105:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177119.738107:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177119.738109:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177119.738112:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177119.738114:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177119.738117:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177119.738124:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856339328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177119.738127:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177119.738129:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738135:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177119.738136:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177119.738138:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177119.738139:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177119.738141:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177119.738144:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856339456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177119.738147:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177119.738148:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177119.738150:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177119.738151:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177119.738154:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7800.
02000000:00000001:0.0:1786177119.738156:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177119.738159:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177119.738161:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177119.738164:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177119.738166:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856339456, portal 4
00000100:00000001:0.0:1786177119.738168:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177119.738169:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454188800 : -114548255362816 : ffff97d1a7edf100)
00000100:00000040:0.0:1786177119.738175:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738180:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177119.738182:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177119.738184:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177119.738186:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856339456, offset 0
00000400:00000200:0.0:1786177119.738189:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177119.738193:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177119.738196:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177119.738199:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177119.738201:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177119.738204:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177119.738206:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177119.738210:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254823:254823:256:4294967295] 192.168.202.110@tcp LPNI seq info [254812:254812:8:4294967295]
00000400:00000200:0.0:1786177119.738214:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177119.738220:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177119.738224:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177119.738227:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177119.738231:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177119.738233:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786177119.738237:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177119.738240:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786177119.738244:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177119.738246:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177119.738248:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177119.738251:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177119.738253:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177119.738254:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177119.738256:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177119.738259:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738271:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177119.738274:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738280:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.738281:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177119.738283:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177119.738286:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177119.738289:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177119.738291:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302015498
00000100:00000001:0.0:1786177119.738293:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302019277
00000100:00000001:0.0:1786177119.738295:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302015498
00000100:00000001:0.0:1786177119.738297:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302019277
00000100:00000001:0.0:1786177119.738299:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177119.738301:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.738303:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177119.738306:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177119/real 1786177119]  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177119.738313:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177119.738315:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177119.738316:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177119.738318:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177119.738320:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177119.738322:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177119.738325:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177119.738332:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738338:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177119.738339:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738341:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177119.738342:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177119.738345:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738351:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177119.738355:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738361:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177119.738363:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738364:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177119.738365:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738367:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177119.738368:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177119.738370:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177119.738373:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177119.738375:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177119.738379:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177119.738385:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856339456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177119.738389:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177119.738391:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738396:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177119.738397:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177119.738399:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177119.738401:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177119.738402:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177119.738405:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856339584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177119.738408:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177119.738409:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177119.738411:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177119.738413:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177119.738415:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177119.738417:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177119.738419:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177119.738421:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177119.738424:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177119.738427:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856339584, portal 4
00000100:00000001:0.0:1786177119.738428:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177119.738430:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00000040:0.0:1786177119.738435:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738440:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177119.738442:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177119.738444:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177119.738445:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856339584, offset 0
00000400:00000200:0.0:1786177119.738449:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177119.738452:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177119.738455:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177119.738457:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177119.738459:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177119.738461:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177119.738464:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177119.738467:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254824:254824:256:4294967295] 192.168.202.110@tcp LPNI seq info [254813:254813:8:4294967295]
00000400:00000200:0.0:1786177119.738471:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177119.738476:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177119.738480:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177119.738483:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177119.738486:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177119.738489:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786177119.738492:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177119.738494:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786177119.738498:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177119.738500:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177119.738502:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177119.738505:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177119.738508:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177119.738537:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177119.738539:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177119.738543:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738551:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177119.738554:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738560:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.738562:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177119.738565:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177119.738567:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177119.738570:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177119.738572:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302015507
00000100:00000001:0.0:1786177119.738575:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302019277
00000100:00000001:0.0:1786177119.738576:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302015507
00000100:00000001:0.0:1786177119.738579:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302019277
00000100:00000001:0.0:1786177119.738580:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177119.738582:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.738584:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177119.738587:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177119/real 1786177119]  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177119.738595:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177119.738597:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177119.738598:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177119.738600:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177119.738602:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177119.738603:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177119.738606:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177119.738614:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738620:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177119.738622:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738623:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177119.738624:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177119.738628:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738634:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177119.738638:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738645:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177119.738647:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738649:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177119.738650:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738652:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177119.738653:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177119.738656:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177119.738658:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177119.738660:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177119.738664:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177119.738678:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856339584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177119.738682:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738683:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177119.738686:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edca80 x1872934856339200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738693:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177119.738694:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177119.738696:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177119.738698:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177119.738699:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177119.738701:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177119.738702:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177119.738704:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:0.0:1786177119.738707:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177119.738708:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177119.738710:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edca80.
00000100:00000001:0.0:1786177119.738712:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177119.738713:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738714:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177119.738717:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856339328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738722:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177119.738723:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177119.738725:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177119.738727:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177119.738728:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177119.738729:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177119.738731:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177119.738732:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:0.0:1786177119.738734:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177119.738735:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177119.738737:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
00000100:00000001:0.0:1786177119.738739:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177119.738740:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738742:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177119.738745:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf100 x1872934856339456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738751:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177119.738753:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177119.738754:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7800.
02000000:00000001:0.0:1786177119.738756:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177119.738757:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177119.738763:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177119.738765:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177119.738767:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:0.0:1786177119.738769:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177119.738770:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177119.738772:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf100.
00000100:00000001:0.0:1786177119.738774:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177119.738775:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738777:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177119.738780:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856339584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177119.738785:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177119.738786:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177119.738788:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177119.738789:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177119.738790:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177119.738792:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177119.738793:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177119.738795:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:0.0:1786177119.738797:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177119.738797:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177119.738798:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:0.0:1786177119.738800:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177119.738801:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738803:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177119.738807:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177119.738808:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177119.738810:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177119.738811:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177120.312601:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177120.312605:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177120.312609:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177120.312610:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177120.312612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177120.312613:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177120.312657:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177120.312661:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177120.312665:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177120.760584:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177120.760591:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177120.760593:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177120.760599:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177120.760603:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177120.760606:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177120.760608:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177120.760611:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177120.760615:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177121.336602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177121.336608:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177121.336609:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177121.336611:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177121.336615:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177121.336617:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177121.336618:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177121.336620:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177121.336622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177121.336623:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177121.336623:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177121.336625:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177121.336626:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177121.336627:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177121.336628:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177121.336629:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177121.336631:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177121.336632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177121.336682:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177121.336685:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177121.336687:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177121.784598:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177121.784605:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177121.784606:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177121.784611:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177121.784614:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177121.784618:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177121.784621:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177121.784624:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177121.784626:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177122.360583:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177122.360588:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177122.360592:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177122.360593:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177122.360595:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177122.360595:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177122.360637:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177122.360640:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177122.360642:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177122.808599:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177122.808607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177122.808609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177122.808614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177122.808617:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177122.808620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177122.808623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177122.808625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177122.808627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177123.384590:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177123.384595:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177123.384599:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177123.384600:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177123.384602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177123.384605:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177123.384606:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177123.384609:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177123.384611:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177123.384613:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177123.384614:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177123.384616:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177123.384674:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177123.384679:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177123.384683:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177123.576586:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302019277, at 4302019328
00000100:00000040:1.0:1786177123.576598:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302019277, at 4302019328
00000100:00000010:0.0:1786177123.576600:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000010:1.0:1786177123.576610:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134ae0.
00000100:00000001:3.0:1786177123.832578:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177123.832584:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177123.832585:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177123.832590:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177123.832593:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177123.832596:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177123.832598:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177123.832600:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177123.832602:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177124.408597:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177124.408604:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177124.408610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177124.408611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177124.408614:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177124.408615:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177124.408688:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177124.408691:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177124.408694:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786177124.856620:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177124.856621:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177124.856628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:2.0:1786177124.856630:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177124.856632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.856635:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1786177124.856639:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786177124.856640:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177124.856646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177124.856648:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177124.856650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177124.856653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177124.856656:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177124.856659:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7348
00000100:00000001:3.0:1786177124.856660:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177124.856671:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177124.856681:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177124.856684:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177124.856687:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.856689:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177124.856692:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.856694:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177124.856696:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177124.856698:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177124.856700:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177124.856703:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177124.856704:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177124.856706:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177124.856709:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.856710:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177124.856712:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.856713:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177124.856715:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177124.856718:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177124.856719:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177124.856721:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177124.856724:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177124.856726:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.856728:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177124.856730:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177124.856746:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786177124.856749:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177124.856752:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177124.856753:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.856756:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177124.856758:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177124.856762:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177124.856769:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786177124.856775:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177124.856782:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177124.856790:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177124.856793:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177124.856797:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177124.856814:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177124.856816:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177124.856819:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177124.856820:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177124.856821:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177124.856823:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177124.856825:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7348
00000400:00000200:2.0:1786177124.856828:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177124.856831:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177124.856833:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177124.856836:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.856837:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177124.856840:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.856842:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177124.856844:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177124.856845:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177124.856846:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786177124.856848:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1786177124.856849:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177124.856854:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786177124.856855:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1786177124.856857:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786177124.856859:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177124.856864:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.856868:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177124.856871:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177124.856874:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.856880:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177124.856886:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177124.856893:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177124.856894:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177124.856897:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.856899:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177124.856903:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177124.856904:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177124.856906:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177124.856909:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177124.856911:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:0.0:1786177124.856913:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856339712:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1786177124.856918:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.856919:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177124.856924:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1786177124.856924:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:0.0:1786177124.856928:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177124.856931:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:2.0:1786177124.856931:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000010:0.0:1786177124.856937:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6800.
00000008:00000001:2.0:1786177124.856937:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177124.856941:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.856943:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177124.856948:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:2.0:1786177124.856950:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000400:00000010:0.0:1786177124.856953:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000020:00000040:2.0:1786177124.856956:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000400:00000010:0.0:1786177124.856960:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
02000000:00000001:2.0:1786177124.856960:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177124.856962:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.856965:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177124.856967:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:0.0:1786177124.856968:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856339712, portal 25
02000000:00000001:2.0:1786177124.856969:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.856974:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177124.856976:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
02000000:00000010:2.0:1786177124.856977:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786177124.856984:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177124.856990:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177124.856997:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:0.0:1786177124.856998:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177124.856999:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177124.857005:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.857025:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786177124.857026:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177124.857030:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1786177124.857034:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786177124.857037:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177124.857040:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:0.0:1786177124.857041:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00080000:2.0:1786177124.857043:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:0.0:1786177124.857047:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856339712, offset 0
00000100:00000001:2.0:1786177124.857049:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177124.857055:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7348
00000400:00000200:0.0:1786177124.857061:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177124.857065:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177124.857071:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177124.857073:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177124.857078:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786177124.857080:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1786177124.857085:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177124.857087:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786177124.857091:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786177124.857094:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:2.0:1786177124.857099:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177124.857100:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177124.857105:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.857106:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786177124.857109:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254825:254825:256:4294967295] 192.168.202.110@tcp LPNI seq info [254814:254814:8:4294967295]
00000020:00000001:2.0:1786177124.857113:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:0.0:1786177124.857115:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786177124.857117:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177124.857122:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177124.857125:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:0.0:1786177124.857126:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000001:2.0:1786177124.857130:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786177124.857134:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:0.0:1786177124.857137:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177124.857141:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000010:0.0:1786177124.857143:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000100:00000001:2.0:1786177124.857145:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177124.857150:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:0.0:1786177124.857153:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177124.857155:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:0.0:1786177124.857157:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000040:2.0:1786177124.857161:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000800:00000200:0.1:1786177124.857165:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177124.857169:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00080000:2.0:1786177124.857170:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000100:0.0:1786177124.857176:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177124.857178:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1786177124.857180:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
02000000:00000001:2.0:1786177124.857185:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786177124.857188:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1786177124.857190:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1786177124.857194:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:2.0:1786177124.857196:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.857199:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:0.0:1786177124.857200:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000008:00100000:2.0:1786177124.857202:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000010:0.0:1786177124.857203:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000008:00000001:2.0:1786177124.857208:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.857211:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.857218:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000010:2.0:1786177124.857219:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000100:00000200:0.0:1786177124.857224:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177124.857225:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177124.857232:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177124.857236:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.857238:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786177124.857240:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:0.0:1786177124.857244:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177124.857245:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177124.857249:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177124.857256:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000001:0.0:1786177124.857257:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.857262:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177124.857264:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.857267:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177124.857272:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1786177124.857272:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177124.857277:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302024397
00000100:00000001:2.0:1786177124.857285:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000010:0.0:1786177124.857287:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:2.0:1786177124.857288:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177124.857293:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.857297:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177124.857301:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177124.857305:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177124.857306:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302024397
00000100:00080000:2.0:1786177124.857309:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177124.857311:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000010:0.0:1786177124.857314:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134180.
00000100:00080000:2.0:1786177124.857314:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177124.857320:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786177124.857322:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00080000:2.0:1786177124.857322:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7348
00000400:00000200:2.0:1786177124.857330:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177124.857331:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1786177124.857335:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1786177124.857336:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.857339:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177124.857344:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177124/real 1786177124]  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786177124.857348:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177124.857356:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000001:0.0:1786177124.857359:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786177124.857359:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:0.0:1786177124.857363:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:2.0:1786177124.857363:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.857367:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000010:0.0:1786177124.857369:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000020:00000001:2.0:1786177124.857369:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177124.857372:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177124.857375:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:0.0:1786177124.857377:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000020:00000040:2.0:1786177124.857377:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000400:00000010:0.0:1786177124.857382:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000020:00000001:2.0:1786177124.857382:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177124.857388:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:0.0:1786177124.857390:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:2.0:1786177124.857391:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.857395:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177124.857399:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:0.0:1786177124.857400:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177124.857401:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177124.857406:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177124.857410:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177124.857416:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177124.857420:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177124.857424:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:0.0:1786177124.857427:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177124.857427:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.857432:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177124.857435:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786177124.857441:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000008:00000001:2.0:1786177124.857441:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.857444:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177124.857445:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.857448:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000010:2.0:1786177124.857450:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000100:00000001:0.0:1786177124.857451:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786177124.857455:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00100000:0.0:1786177124.857458:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177124.857459:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1786177124.857484:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177124.857488:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177124.857492:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.857497:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786177124.857498:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:2.0:1786177124.857500:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1786177124.857503:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786177124.857505:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.857506:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177124.857537:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.857541:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177124.857544:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000010:2.0:1786177124.857545:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00080000:0.0:1786177124.857549:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:2.0:1786177124.857552:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177124.857556:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177124.857559:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00100000:2.0:1786177124.857564:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1786177124.857567:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177124.857597:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00100000:0.0:1786177124.857599:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856339712:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177124.857601:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.857610:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177124.857610:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.857617:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786177124.857622:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177124.857626:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1786177124.857627:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856339712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786177124.857631:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177124.857636:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177124.857639:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177124.857642:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000001:2.0:1786177124.857644:0:24919:0:(import.c:485:import_select_connection()) Process entered
02000000:00000001:0.0:1786177124.857645:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177124.857649:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6800.
00000100:00080000:2.0:1786177124.857652:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7348
02000000:00000001:0.0:1786177124.857654:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177124.857657:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177124.857659:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177124.857663:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000400:00000200:2.0:1786177124.857663:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1786177124.857667:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
00000400:00000200:2.0:1786177124.857672:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:0.0:1786177124.857673:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177124.857677:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00080000:2.0:1786177124.857678:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000010:0.0:1786177124.857679:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:0.0:1786177124.857684:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000040:2.0:1786177124.857685:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177124.857686:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.857691:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177124.857691:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177124.857696:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.857698:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177124.857700:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000020:00000001:2.0:1786177124.857700:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:0.0:1786177124.857704:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786177124.857704:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177124.857708:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786177124.857709:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177124.857712:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1786177124.857712:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:0.0:1786177124.857715:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1786177124.857716:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786177124.857721:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177124.857721:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177124.857727:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177124.857732:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177124.857736:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177124.857738:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177124.857739:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177124.857741:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786177124.857744:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
02000000:00000001:0.0:1786177124.857745:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177124.857749:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:2.0:1786177124.857750:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1786177124.857752:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.857754:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177124.857758:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856339840:192.168.202.110@tcp:38:kworker.0
02000000:00000001:2.0:1786177124.857759:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177124.857763:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177124.857764:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177124.857766:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177124.857768:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177124.857769:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.857773:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:0.0:1786177124.857774:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:2.0:1786177124.857778:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000010:0.0:1786177124.857779:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6800.
00000008:00000001:2.0:1786177124.857781:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177124.857783:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177124.857784:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177124.857788:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:2.0:1786177124.857791:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000400:00000010:0.0:1786177124.857793:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000020:00000040:2.0:1786177124.857794:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000400:00000010:0.0:1786177124.857798:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
02000000:00000001:2.0:1786177124.857798:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177124.857802:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786177124.857803:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856339840, portal 10
00000100:00000001:2.0:1786177124.857804:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786177124.857806:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
02000000:00000001:2.0:1786177124.857807:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1786177124.857809:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
02000000:00000001:2.0:1786177124.857811:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177124.857816:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1786177124.857816:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786177124.857822:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.857827:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177124.857832:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00100000:2.0:1786177124.857832:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1786177124.857837:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177124.857841:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856339840, offset 0
00000400:00000200:0.0:1786177124.857852:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177124.857856:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1786177124.857860:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177124.857862:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177124.857868:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1786177124.857871:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177124.857874:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177124.857881:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177124.857887:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1786177124.857893:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177124.857894:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177124.857899:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254826:254826:256:4294967295] 192.168.202.110@tcp LPNI seq info [254815:254815:8:4294967295]
00000100:00000040:2.0:1786177124.857902:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7358)
00000400:00000200:0.0:1786177124.857904:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177124.857912:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177124.857916:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177124.857919:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177124.857922:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177124.857924:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786177124.857927:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177124.857929:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786177124.857932:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177124.857933:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177124.857935:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177124.857938:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177124.857940:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177124.857941:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177124.857943:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177124.857946:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.857952:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177124.857954:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.857959:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.857960:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177124.857962:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177124.857966:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177124.857968:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177124.857969:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302020618
00000100:00000001:0.0:1786177124.857971:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302024397
00000100:00000001:0.0:1786177124.857973:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302020618
00000100:00000001:0.0:1786177124.857974:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302024397
00000100:00000001:0.0:1786177124.857976:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177124.857977:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.857978:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177124.857981:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177124/real 1786177124]  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177124.857986:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177124.857988:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177124.857990:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177124.857992:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177124.857994:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177124.857996:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177124.857999:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177124.858008:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858015:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177124.858017:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858019:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177124.858020:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177124.858024:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858031:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177124.858035:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858043:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177124.858046:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858047:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177124.858049:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858051:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177124.858052:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177124.858055:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177124.858058:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177124.858061:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177124.858064:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177124.858069:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856339840:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177124.858072:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177124.858074:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858079:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177124.858080:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177124.858081:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177124.858083:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177124.858084:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177124.858086:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856339968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177124.858088:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177124.858089:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177124.858090:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177124.858091:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177124.858094:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177124.858095:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177124.858098:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177124.858100:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177124.858102:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177124.858104:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856339968, portal 10
00000100:00000001:0.0:1786177124.858105:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177124.858106:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:0.0:1786177124.858110:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858115:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177124.858116:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177124.858118:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177124.858119:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856339968, offset 0
00000400:00000200:0.0:1786177124.858122:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177124.858124:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177124.858127:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177124.858129:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177124.858131:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177124.858132:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177124.858134:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177124.858136:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254827:254827:256:4294967295] 192.168.202.110@tcp LPNI seq info [254816:254816:8:4294967295]
00000400:00000200:0.0:1786177124.858139:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177124.858143:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177124.858146:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177124.858148:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177124.858151:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177124.858153:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786177124.858155:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177124.858157:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786177124.858160:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177124.858162:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177124.858164:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177124.858167:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177124.858169:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177124.858171:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177124.858173:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177124.858176:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858182:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177124.858184:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858189:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.858190:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177124.858192:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177124.858194:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177124.858196:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177124.858197:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302020743
00000100:00000001:0.0:1786177124.858199:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302024397
00000100:00000001:0.0:1786177124.858200:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302020743
00000100:00000001:0.0:1786177124.858202:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302024397
00000100:00000001:0.0:1786177124.858203:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177124.858204:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.858206:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177124.858208:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177124/real 1786177124]  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177124.858213:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177124.858213:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177124.858214:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177124.858215:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177124.858216:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177124.858218:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177124.858219:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177124.858225:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858229:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177124.858230:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858231:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177124.858232:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177124.858235:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858238:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177124.858241:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858245:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177124.858247:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858248:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177124.858249:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858250:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177124.858251:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177124.858253:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177124.858255:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177124.858257:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177124.858260:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177124.858265:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856339968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177124.858268:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177124.858270:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858274:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177124.858275:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177124.858277:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177124.858278:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177124.858280:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177124.858283:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856340096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177124.858286:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177124.858288:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177124.858289:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177124.858291:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177124.858294:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177124.858296:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177124.858299:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177124.858302:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177124.858305:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177124.858308:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856340096, portal 4
00000100:00000001:0.0:1786177124.858311:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177124.858312:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000040:0.0:1786177124.858319:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858326:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177124.858329:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177124.858331:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177124.858333:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856340096, offset 0
00000400:00000200:0.0:1786177124.858338:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177124.858341:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177124.858343:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177124.858345:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177124.858347:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177124.858349:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177124.858350:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177124.858353:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254828:254828:256:4294967295] 192.168.202.110@tcp LPNI seq info [254817:254817:8:4294967295]
00000400:00000200:0.0:1786177124.858355:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177124.858360:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177124.858364:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177124.858367:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177124.858371:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177124.858374:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786177124.858378:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177124.858381:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786177124.858384:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177124.858386:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177124.858389:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177124.858393:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177124.858396:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177124.858398:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177124.858401:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177124.858405:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858414:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177124.858416:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858421:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.858422:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177124.858424:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177124.858427:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177124.858429:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177124.858431:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302020618
00000100:00000001:0.0:1786177124.858432:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302024397
00000100:00000001:0.0:1786177124.858434:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302020618
00000100:00000001:0.0:1786177124.858435:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302024397
00000100:00000001:0.0:1786177124.858437:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177124.858438:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.858439:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177124.858441:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177124/real 1786177124]  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177124.858447:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177124.858448:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177124.858449:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177124.858451:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177124.858452:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177124.858453:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177124.858455:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177124.858475:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858480:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177124.858481:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858482:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177124.858483:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177124.858486:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858490:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177124.858493:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858497:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177124.858499:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858501:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177124.858502:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858504:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177124.858505:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177124.858507:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177124.858530:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177124.858533:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177124.858539:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177124.858548:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856340096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177124.858552:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858555:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177124.858557:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856339840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858562:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177124.858563:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177124.858565:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177124.858567:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177124.858568:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177124.858569:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177124.858570:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177124.858571:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:0.0:1786177124.858574:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177124.858575:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177124.858576:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:0.0:1786177124.858578:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177124.858579:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858580:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177124.858582:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856339968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858587:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177124.858588:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177124.858589:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177124.858590:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177124.858592:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177124.858593:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177124.858595:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177124.858596:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:0.0:1786177124.858598:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177124.858600:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177124.858601:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:0.0:1786177124.858603:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177124.858605:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858606:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177124.858610:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856340096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858617:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177124.858618:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177124.858620:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177124.858622:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177124.858624:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177124.858625:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177124.858627:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177124.858629:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:0.0:1786177124.858631:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177124.858633:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177124.858634:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:0.0:1786177124.858637:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177124.858638:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858640:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858644:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177124.858645:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.858646:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177124.858647:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177124.858648:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177124.858651:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858656:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177124.858657:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177124.858659:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177124.858660:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177124.858661:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177124.858663:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856340224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177124.858665:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177124.858667:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177124.858668:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177124.858669:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177124.858672:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177124.858673:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177124.858676:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177124.858677:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000400:00000010:0.0:1786177124.858679:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00000200:0.0:1786177124.858682:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856340224, portal 4
00000100:00000001:0.0:1786177124.858683:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177124.858684:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:0.0:1786177124.858688:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858693:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177124.858694:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177124.858695:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00000200:0.0:1786177124.858697:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856340224, offset 0
00000400:00000200:0.0:1786177124.858700:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177124.858702:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177124.858704:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177124.858706:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177124.858708:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177124.858710:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177124.858711:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177124.858713:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254829:254829:256:4294967295] 192.168.202.110@tcp LPNI seq info [254818:254818:8:4294967295]
00000400:00000200:0.0:1786177124.858716:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177124.858721:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177124.858724:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177124.858726:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
00000800:00000200:0.0:1786177124.858728:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177124.858730:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786177124.858733:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177124.858735:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786177124.858737:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177124.858739:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:0.0:1786177124.858741:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177124.858743:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177124.858745:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:0.0:1786177124.858746:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:0.0:1786177124.858748:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177124.858750:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858755:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177124.858757:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858762:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.858763:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177124.858764:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177124.858767:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177124.858769:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177124.858770:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302020627
00000100:00000001:0.0:1786177124.858771:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302024397
00000100:00000001:0.0:1786177124.858773:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302020627
00000100:00000001:0.0:1786177124.858775:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302024397
00000100:00000001:0.0:1786177124.858776:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177124.858777:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.858778:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177124.858780:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177124/real 1786177124]  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177124.858785:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177124.858786:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177124.858787:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177124.858789:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177124.858790:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177124.858792:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177124.858794:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177124.858799:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858803:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177124.858804:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858805:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177124.858806:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177124.858808:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858812:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177124.858814:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858818:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177124.858820:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858821:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177124.858822:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858823:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177124.858824:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177124.858826:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177124.858827:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177124.858829:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177124.858831:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177124.858836:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856340224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177124.858838:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858839:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177124.858842:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856340224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177124.858849:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177124.858850:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177124.858852:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177124.858854:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177124.858856:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177124.858857:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177124.858859:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177124.858861:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:0.0:1786177124.858863:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177124.858865:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177124.858867:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:0.0:1786177124.858869:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177124.858871:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858873:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177124.858876:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177124.858878:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177124.858880:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177124.858881:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177125.432587:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177125.432593:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177125.432598:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177125.432599:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177125.432602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177125.432603:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177125.432663:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177125.432666:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177125.432669:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177125.880584:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177125.880591:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177125.880593:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177125.880597:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177125.880600:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177125.880603:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177125.880605:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177125.880606:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177125.880608:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177126.456633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177126.456638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177126.456642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177126.456643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177126.456645:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177126.456646:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177126.456696:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177126.456700:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177126.456703:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177126.904630:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177126.904636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177126.904637:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177126.904641:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177126.904645:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177126.904648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177126.904650:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177126.904652:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177126.904654:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177127.480601:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177127.480606:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177127.480610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177127.480611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177127.480614:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177127.480614:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177127.480676:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177127.480679:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177127.480681:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177127.928595:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177127.928602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177127.928605:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177127.928610:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177127.928614:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177127.928618:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177127.928621:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177127.928624:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177127.928627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177128.504693:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177128.504701:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177128.504705:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177128.504707:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177128.504709:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177128.504710:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177128.504809:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177128.504815:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177128.504819:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177128.696672:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302024397, at 4302024448
00000100:00000010:1.0:1786177128.696693:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134180.
00000100:00000040:0.0:1786177128.696817:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302024397, at 4302024448
00000100:00000010:0.0:1786177128.696824:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:3.0:1786177128.952657:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177128.952666:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177128.952668:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177128.952675:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177128.952678:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177128.952682:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177128.952685:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177128.952687:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177128.952690:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177129.528588:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177129.528597:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177129.528602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177129.528603:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177129.528606:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177129.528607:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177129.528655:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177129.528660:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177129.528664:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177129.976575:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177129.976584:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177129.976586:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177129.976592:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177129.976597:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177129.976601:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.976605:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177129.976608:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.976612:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177129.976630:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177129.976637:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177129.976640:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177129.976645:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177129.976649:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177129.976657:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7353
00000400:00000200:0.0:1786177129.976667:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177129.976675:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177129.976678:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177129.976682:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177129.976686:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177129.976689:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177129.976692:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177129.976695:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177129.976697:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177129.976701:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177129.976704:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177129.976707:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177129.976709:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177129.976712:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177129.976715:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177129.976717:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177129.976719:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177129.976722:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177129.976725:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177129.976727:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177129.976729:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177129.976732:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177129.976734:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177129.976735:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177129.976737:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177129.976745:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419c00.
00000020:00000040:0.0:1786177129.976747:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177129.976751:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177129.976752:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177129.976755:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177129.976757:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177129.976760:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177129.976766:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000001:0.0:1786177129.976773:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177129.976783:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177129.976795:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177129.976799:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177129.976804:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cf890c1c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177129.976839:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177129.976843:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177129.976847:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177129.976848:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177129.976850:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177129.976853:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177129.976856:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7353
00000400:00000200:0.0:1786177129.976862:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177129.976867:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177129.976867:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177129.976872:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1786177129.976873:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177129.976875:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177129.976879:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177129.976884:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177129.976884:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177129.976891:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177129.976896:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177129.976899:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000001:0.0:1786177129.976901:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177129.976902:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177129.976905:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
02000000:00000001:3.0:1786177129.976907:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:0.0:1786177129.976909:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
02000000:00000001:3.0:1786177129.976910:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177129.976914:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177129.976915:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00100000:3.0:1786177129.976918:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856340352:192.168.202.110@tcp:250:kworker.0
00000020:00000001:0.0:1786177129.976921:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:3.0:1786177129.976923:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:0.0:1786177129.976928:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:3.0:1786177129.976929:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177129.976931:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177129.976938:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177129.976942:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1786177129.976942:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:0.0:1786177129.976947:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000010:3.0:1786177129.976949:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:0.0:1786177129.976950:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177129.976952:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177129.976958:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000400:00000200:3.0:1786177129.976962:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177129.976967:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:3.0:1786177129.976968:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000001:0.0:1786177129.976972:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177129.976976:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1786177129.976976:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
02000000:00000001:0.0:1786177129.976982:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:3.0:1786177129.976984:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856340352, portal 25
02000000:00000001:0.0:1786177129.976989:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.976989:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177129.976992:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761661952 : -114547947889664 : ffff97d1ba419c00)
00000100:00000001:0.0:1786177129.976995:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:3.0:1786177129.976999:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1786177129.977000:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177129.977008:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177129.977015:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.977015:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177129.977020:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000010:0.0:1786177129.977023:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41bb80.
00000400:00000010:3.0:1786177129.977024:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000020:00000040:0.0:1786177129.977031:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000200:3.0:1786177129.977031:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856340352, offset 0
02000000:00000001:0.0:1786177129.977038:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1786177129.977042:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786177129.977043:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177129.977048:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177129.977050:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1786177129.977051:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786177129.977054:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177129.977058:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000010:0.0:1786177129.977061:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6000.
00000400:00000200:3.0:1786177129.977064:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1786177129.977070:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177129.977076:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00100000:0.0:1786177129.977080:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177129.977086:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177129.977092:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1786177129.977099:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177129.977099:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254830:254830:256:4294967295] 192.168.202.110@tcp LPNI seq info [254819:254819:8:4294967295]
00000100:00000001:0.0:1786177129.977106:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177129.977107:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177129.977115:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9983a05] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177129.977129:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177129.977137:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177129.977143:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177129.977144:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177129.977151:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177129.977153:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000010:3.0:1786177129.977153:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00080000:0.0:1786177129.977158:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:3.0:1786177129.977164:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177129.977170:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:0.0:1786177129.977175:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:3.1:1786177129.977180:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:0.0:1786177129.977184:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7353
00000800:00000200:3.1:1786177129.977186:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000400:00000200:0.0:1786177129.977193:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000100:3.0:1786177129.977199:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177129.977203:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000010:3.0:1786177129.977205:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00080000:0.0:1786177129.977207:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177129.977213:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177129.977213:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1786177129.977217:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177129.977224:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177129.977227:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1786177129.977229:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177129.977233:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000020:00000001:0.0:1786177129.977234:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177129.977238:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000010:3.0:1786177129.977239:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000020:00000001:0.0:1786177129.977240:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177129.977243:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000001:3.0:1786177129.977245:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:0.0:1786177129.977246:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177129.977249:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000200:3.0:1786177129.977251:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177129.977255:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177129.977260:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177129.977264:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177129.977267:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177129.977268:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1786177129.977272:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000040:3.0:1786177129.977274:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177129.977277:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177129.977283:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177129.977288:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177129.977290:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177129.977292:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177129.977293:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:0.0:1786177129.977297:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177129.977299:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177129.977300:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1786177129.977301:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177129.977306:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177129.977308:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177129.977311:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.977313:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302029517
00000100:00000010:0.0:1786177129.977319:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418700.
00000100:00000010:3.0:1786177129.977321:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad376c0.
00000020:00000040:0.0:1786177129.977326:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000001:3.0:1786177129.977330:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:0.0:1786177129.977332:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177129.977336:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.977336:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302029517
00000100:00000001:0.0:1786177129.977339:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:3.0:1786177129.977340:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37000.
02000000:00000001:0.0:1786177129.977343:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177129.977344:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:0.0:1786177129.977349:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.977349:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177129.977354:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.977357:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000010:0.0:1786177129.977359:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00100000:3.0:1786177129.977361:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177129/real 1786177129]  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177129.977366:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177129.977370:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177129.977372:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177129.977374:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00100000:0.0:1786177129.977377:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177129.977378:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177129.977381:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177129.977385:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177129.977390:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177129.977398:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:3.0:1786177129.977403:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177129.977404:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177129.977429:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bf66f89d] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.977431:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177129.977434:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.977438:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177129.977440:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177129.977448:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177129.977449:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177129.977456:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177129.977459:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177129.977464:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000040:3.0:1786177129.977465:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177129.977470:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177129.977474:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177129.977475:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177129.977481:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177129.977481:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.977485:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177129.977487:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177129.977489:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7353
00000100:00000001:3.0:1786177129.977490:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177129.977492:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177129.977496:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000400:00000200:0.0:1786177129.977499:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1786177129.977502:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177129.977505:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000400:00000200:0.0:1786177129.977507:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1786177129.977528:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177129.977534:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856340352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177129.977538:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.977540:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177129.977543:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419c00 x1872934856340352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177129.977547:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177129.977554:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000040:0.0:1786177129.977556:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177129.977557:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:0.0:1786177129.977560:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000010:3.0:1786177129.977562:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
00000100:00000040:0.0:1786177129.977566:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177129.977567:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:0.0:1786177129.977569:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177129.977570:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000001:0.0:1786177129.977574:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177129.977575:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000040:0.0:1786177129.977577:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177129.977578:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000020:00000001:0.0:1786177129.977582:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
02000000:00000010:3.0:1786177129.977583:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
00000020:00000040:0.0:1786177129.977585:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
02000000:00000001:3.0:1786177129.977587:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000020:00000001:0.0:1786177129.977588:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
02000000:00000001:3.0:1786177129.977590:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000020:00000001:0.0:1786177129.977594:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000010:3.0:1786177129.977594:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419c00.
00000100:00000001:3.0:1786177129.977600:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000040:0.0:1786177129.977601:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177129.977602:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177129.977606:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177129.977607:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177129.977610:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177129.977613:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177129.977614:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:3.0:1786177129.977616:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786177129.977617:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177129.977620:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177129.977621:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177129.977627:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177129.977628:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:0.0:1786177129.977630:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177129.977632:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
02000000:00000001:0.0:1786177129.977633:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.977635:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1786177129.977636:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177129.977640:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177129.977644:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177129.977645:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177129.977646:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177129.977651:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418000.
00000020:00000040:0.0:1786177129.977656:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177129.977661:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177129.977663:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1786177129.977664:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177129.977669:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177129.977669:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177129.977672:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177129.977674:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177129.977675:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177129.977676:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000010:0.0:1786177129.977680:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4c00.
02000000:00000001:3.0:1786177129.977680:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177129.977686:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856340480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177129.977687:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.977691:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177129.977693:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177129.977695:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177129.977697:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1786177129.977698:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177129.977703:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:0.0:1786177129.977706:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1786177129.977708:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177129.977709:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177129.977714:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002e10e4f5] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177129.977717:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177129.977724:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000001:0.0:1786177129.977726:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177129.977731:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1786177129.977731:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00080000:0.0:1786177129.977737:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177129.977739:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:3.0:1786177129.977739:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856340480, portal 10
00000100:00080000:0.0:1786177129.977742:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177129.977744:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177129.977748:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761670016 : -114547947881600 : ffff97d1ba41bb80)
00000100:00000001:0.0:1786177129.977749:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177129.977754:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7353
00000100:00000040:3.0:1786177129.977759:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177129.977762:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177129.977771:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177129.977772:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:0.0:1786177129.977777:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177129.977780:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:0.0:1786177129.977785:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:3.0:1786177129.977786:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000001:0.0:1786177129.977791:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1786177129.977793:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856340480, offset 0
00000100:00000040:0.0:1786177129.977797:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177129.977803:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177129.977803:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:0.0:1786177129.977809:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177129.977812:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000400:00000200:3.0:1786177129.977813:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:0.0:1786177129.977817:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:3.0:1786177129.977820:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:0.0:1786177129.977823:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000400:00000200:3.0:1786177129.977825:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:0.0:1786177129.977829:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:3.0:1786177129.977830:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:0.0:1786177129.977836:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:3.0:1786177129.977836:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177129.977839:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:0.0:1786177129.977847:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177129.977848:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254831:254831:256:4294967295] 192.168.202.110@tcp LPNI seq info [254820:254820:8:4294967295]
00000100:00000001:0.0:1786177129.977855:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177129.977856:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177129.977862:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177129.977868:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177129.977869:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.0:1786177129.977875:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:0.0:1786177129.977876:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000800:00000010:3.0:1786177129.977880:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00080000:0.0:1786177129.977885:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:3.0:1786177129.977888:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177129.977892:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177129.977893:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
02000000:00000001:0.0:1786177129.977899:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:3.1:1786177129.977900:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177129.977905:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
02000000:00000001:0.0:1786177129.977906:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000100:3.0:1786177129.977909:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786177129.977914:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1786177129.977914:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177129.977918:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1786177129.977920:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:3.0:1786177129.977924:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000008:00100000:0.0:1786177129.977926:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:3.0:1786177129.977927:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177129.977929:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000008:00000001:0.0:1786177129.977935:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.977935:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177129.977940:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177129.977941:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177129.977950:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b800.
00000100:00000001:3.0:1786177129.977950:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177129.977955:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177129.977957:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177129.977964:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177129.977966:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177129.977967:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.977969:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177129.977974:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177129.977974:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177129.977978:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:3.0:1786177129.977980:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
02000000:00000001:0.0:1786177129.977982:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177129.977984:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177129.977988:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302025738
02000000:00000010:0.0:1786177129.977990:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000100:00000001:3.0:1786177129.977991:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302029517
00000100:00000001:3.0:1786177129.977996:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302025738
00000100:00000001:0.0:1786177129.977998:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.977999:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302029517
00000100:00000001:3.0:1786177129.978004:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177129.978006:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.978008:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177129.978010:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1786177129.978013:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177129/real 1786177129]  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177129.978034:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000001:3.0:1786177129.978036:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177129.978038:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1786177129.978040:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177129.978041:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177129.978044:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177129.978047:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177129.978049:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:0.0:1786177129.978053:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000192ee9a0] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177129.978054:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177129.978074:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177129.978084:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.978084:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177129.978087:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978089:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177129.978090:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177129.978093:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7363)
00000100:00100000:3.0:1786177129.978094:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978101:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177129.978104:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978109:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177129.978111:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978113:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177129.978114:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978117:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177129.978119:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177129.978122:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177129.978126:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177129.978129:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177129.978133:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177129.978141:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856340480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177129.978146:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177129.978149:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978156:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177129.978158:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177129.978161:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177129.978163:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177129.978165:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177129.978168:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856340608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177129.978171:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177129.978173:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177129.978175:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177129.978177:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177129.978181:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177129.978183:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177129.978185:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177129.978187:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177129.978189:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177129.978191:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856340608, portal 10
00000100:00000001:3.0:1786177129.978193:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177129.978194:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761656576 : -114547947895040 : ffff97d1ba418700)
00000100:00000040:3.0:1786177129.978198:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978203:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177129.978204:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177129.978206:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177129.978207:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856340608, offset 0
00000400:00000200:3.0:1786177129.978210:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177129.978212:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177129.978215:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177129.978217:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177129.978219:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177129.978220:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177129.978222:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177129.978224:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254832:254832:256:4294967295] 192.168.202.110@tcp LPNI seq info [254821:254821:8:4294967295]
00000400:00000200:3.0:1786177129.978227:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177129.978231:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177129.978234:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177129.978236:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177129.978239:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177129.978241:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786177129.978244:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177129.978245:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1786177129.978283:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177129.978284:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177129.978287:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177129.978289:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177129.978291:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177129.978292:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177129.978294:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177129.978296:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978302:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177129.978304:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978312:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.978314:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177129.978316:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177129.978318:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177129.978320:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177129.978322:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302025863
00000100:00000001:3.0:1786177129.978323:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302029517
00000100:00000001:3.0:1786177129.978325:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302025863
00000100:00000001:3.0:1786177129.978326:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302029517
00000100:00000001:3.0:1786177129.978328:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177129.978329:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.978330:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177129.978332:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177129/real 1786177129]  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177129.978337:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177129.978338:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177129.978339:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177129.978341:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177129.978342:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177129.978343:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177129.978345:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177129.978351:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978354:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177129.978355:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978356:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177129.978357:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177129.978359:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978364:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177129.978367:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978371:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177129.978372:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978373:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177129.978374:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978375:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177129.978376:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177129.978378:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177129.978380:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177129.978381:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177129.978383:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177129.978388:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856340608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177129.978390:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978392:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177129.978394:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41bb80 x1872934856340480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978398:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177129.978399:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177129.978400:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177129.978402:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177129.978403:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177129.978404:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177129.978405:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177129.978406:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6000.
02000000:00000001:3.0:1786177129.978409:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177129.978410:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177129.978425:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41bb80.
00000100:00000001:3.0:1786177129.978427:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177129.978428:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978429:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177129.978432:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418700 x1872934856340608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978436:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177129.978437:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177129.978438:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177129.978440:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177129.978441:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177129.978442:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177129.978443:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177129.978444:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:3.0:1786177129.978445:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177129.978446:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177129.978447:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418700.
00000100:00000001:3.0:1786177129.978449:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177129.978450:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978451:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978454:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177129.978455:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.978456:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177129.978457:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177129.978458:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177129.978460:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978464:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177129.978465:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177129.978467:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177129.978468:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177129.978470:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177129.978472:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856340736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177129.978475:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177129.978477:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177129.978478:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177129.978479:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177129.978481:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177129.978482:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177129.978485:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177129.978486:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177129.978489:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177129.978491:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856340736, portal 4
00000100:00000001:3.0:1786177129.978493:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177129.978494:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761654784 : -114547947896832 : ffff97d1ba418000)
00000100:00000040:3.0:1786177129.978498:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978502:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177129.978503:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177129.978505:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177129.978506:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856340736, offset 0
00000400:00000200:3.0:1786177129.978509:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177129.978527:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177129.978529:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177129.978531:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177129.978533:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177129.978535:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177129.978537:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177129.978539:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254833:254833:256:4294967295] 192.168.202.110@tcp LPNI seq info [254822:254822:8:4294967295]
00000400:00000200:3.0:1786177129.978541:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177129.978546:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177129.978549:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177129.978551:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177129.978554:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177129.978556:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786177129.978559:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177129.978561:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1786177129.978563:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177129.978564:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177129.978566:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177129.978569:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177129.978571:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177129.978572:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177129.978574:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177129.978576:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978581:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177129.978583:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978588:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.978589:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177129.978591:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177129.978593:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177129.978595:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177129.978596:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302025739
00000100:00000001:3.0:1786177129.978598:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302029517
00000100:00000001:3.0:1786177129.978600:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302025739
00000100:00000001:3.0:1786177129.978601:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302029517
00000100:00000001:3.0:1786177129.978602:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177129.978603:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.978605:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177129.978607:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177129/real 1786177129]  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177129.978611:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177129.978612:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177129.978614:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177129.978617:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177129.978619:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177129.978621:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177129.978624:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177129.978633:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978640:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177129.978641:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978643:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177129.978645:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177129.978649:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978657:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177129.978661:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978669:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177129.978671:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978673:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177129.978674:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978675:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177129.978676:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177129.978678:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177129.978680:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177129.978682:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177129.978685:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177129.978690:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856340736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177129.978693:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177129.978695:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978700:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177129.978701:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177129.978703:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177129.978704:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177129.978706:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177129.978709:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856340864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177129.978712:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177129.978714:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177129.978715:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177129.978717:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177129.978720:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177129.978723:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177129.978727:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177129.978729:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177129.978733:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177129.978736:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856340864, portal 4
00000100:00000001:3.0:1786177129.978738:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177129.978740:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761669120 : -114547947882496 : ffff97d1ba41b800)
00000100:00000040:3.0:1786177129.978746:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978753:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177129.978755:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177129.978757:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177129.978759:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856340864, offset 0
00000400:00000200:3.0:1786177129.978761:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177129.978765:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177129.978769:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177129.978772:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177129.978775:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177129.978777:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177129.978779:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177129.978782:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254834:254834:256:4294967295] 192.168.202.110@tcp LPNI seq info [254823:254823:8:4294967295]
00000400:00000200:3.0:1786177129.978787:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177129.978794:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177129.978799:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177129.978803:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177129.978807:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177129.978809:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786177129.978814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177129.978816:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1786177129.978819:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177129.978820:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177129.978823:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177129.978825:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177129.978827:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177129.978829:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177129.978830:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177129.978833:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978839:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177129.978841:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978845:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.978847:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177129.978849:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177129.978851:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177129.978854:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177129.978855:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302025747
00000100:00000001:3.0:1786177129.978857:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302029517
00000100:00000001:3.0:1786177129.978859:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302025747
00000100:00000001:3.0:1786177129.978860:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302029517
00000100:00000001:3.0:1786177129.978862:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177129.978863:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.978865:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177129.978867:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177129/real 1786177129]  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177129.978871:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177129.978872:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177129.978873:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177129.978875:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177129.978876:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177129.978877:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177129.978879:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177129.978884:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978888:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177129.978889:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978890:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177129.978891:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177129.978893:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978897:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177129.978900:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978904:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177129.978905:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978906:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177129.978907:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978909:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177129.978910:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177129.978912:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177129.978914:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177129.978916:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177129.978918:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177129.978922:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856340864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177129.978924:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978926:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177129.978928:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418000 x1872934856340736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978931:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177129.978933:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177129.978934:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177129.978935:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177129.978937:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177129.978937:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177129.978939:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177129.978940:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:3.0:1786177129.978942:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177129.978942:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177129.978944:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418000.
00000100:00000001:3.0:1786177129.978945:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177129.978946:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978948:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177129.978949:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b800 x1872934856340864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177129.978953:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177129.978954:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177129.978955:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177129.978956:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177129.978957:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177129.978958:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177129.978959:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177129.978960:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:3.0:1786177129.978961:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177129.978962:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177129.978963:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b800.
00000100:00000001:3.0:1786177129.978964:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177129.978965:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978966:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177129.978968:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177129.978969:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177129.978971:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177129.978972:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177130.552629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177130.552638:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177130.552640:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177130.552644:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177130.552649:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177130.552652:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177130.552654:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177130.552657:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177130.552661:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177130.552663:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177130.552665:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177130.552666:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177130.552667:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177130.552668:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177130.552670:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177130.552671:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177130.552674:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177130.552674:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177130.552734:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177130.552737:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177130.552740:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177131.000563:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177131.000569:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177131.000571:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177131.000578:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177131.000581:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177131.000586:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177131.000588:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177131.000592:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177131.000595:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177131.576644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177131.576650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177131.576658:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177131.576659:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177131.576663:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177131.576665:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177131.576732:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177131.576735:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177131.576738:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177132.024604:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177132.024611:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177132.024613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177132.024617:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177132.024620:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177132.024624:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177132.024626:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177132.024628:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177132.024631:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177132.600611:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177132.600617:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177132.600622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177132.600623:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177132.600625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177132.600628:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177132.600630:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177132.600633:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177132.600636:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177132.600639:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177132.600641:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177132.600643:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177132.600701:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177132.600704:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177132.600707:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177133.048615:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177133.048622:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177133.048625:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177133.048631:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177133.048635:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177133.048640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177133.048642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177133.048646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177133.048648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177133.624609:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177133.624616:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177133.624622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177133.624623:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177133.624626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177133.624626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177133.624686:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177133.624690:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177133.624694:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177133.816589:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302029517, at 4302029568
00000100:00000040:1.0:1786177133.816589:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302029517, at 4302029568
00000100:00000010:1.0:1786177133.816604:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37000.
00000100:00000010:0.0:1786177133.816606:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad376c0.
00000100:00000001:3.0:1786177134.072602:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177134.072608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177134.072610:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177134.072615:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177134.072618:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177134.072621:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177134.072623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177134.072625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177134.072627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177134.648639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177134.648647:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177134.648655:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177134.648656:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177134.648660:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177134.648662:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177134.648728:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177134.648732:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177134.648735:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786177135.096618:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177135.096629:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177135.096632:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177135.096636:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177135.096641:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177135.096647:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7358
00000100:00000001:0.0:1786177135.096655:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000400:00000200:3.0:1786177135.096656:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177135.096660:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177135.096663:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177135.096666:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00100000:0.0:1786177135.096668:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:3.0:1786177135.096670:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177135.096672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1786177135.096675:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177135.096677:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177135.096679:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177135.096681:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:3.0:1786177135.096683:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177135.096685:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177135.096686:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177135.096689:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177135.096690:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177135.096694:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177135.096697:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177135.096699:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177135.096702:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177135.096704:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177135.096706:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177135.096709:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177135.096710:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177135.096712:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177135.096715:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177135.096718:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177135.096719:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177135.096721:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177135.096724:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177135.096726:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.096728:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177135.096729:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177135.096737:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f800.
00000020:00000040:3.0:1786177135.096740:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177135.096743:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177135.096744:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.096747:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177135.096749:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177135.096752:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177135.096759:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786177135.096764:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177135.096773:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177135.096784:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177135.096789:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177135.096794:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4d339d9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177135.096820:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177135.096823:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177135.096825:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177135.096826:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177135.096827:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177135.096830:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177135.096832:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7358
00000400:00000200:3.0:1786177135.096835:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177135.096838:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177135.096839:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177135.096842:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177135.096844:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177135.096846:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177135.096847:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177135.096850:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177135.096851:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177135.096852:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177135.096854:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:0.0:1786177135.096855:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1786177135.096856:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786177135.096860:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1786177135.096860:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786177135.096864:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177135.096864:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177135.096867:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177135.096873:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:0.0:1786177135.096874:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177135.096875:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177135.096881:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786177135.096887:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177135.096891:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177135.096893:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1786177135.096894:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177135.096895:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177135.096899:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177135.096900:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177135.096901:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177135.096903:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177135.096905:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.096906:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00100000:0.0:1786177135.096912:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856340992:192.168.202.110@tcp:250:kworker.0
00000008:00100000:3.0:1786177135.096912:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177135.096917:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177135.096919:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1786177135.096920:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177135.096924:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:3.0:1786177135.096925:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69fb80.
02000000:00000001:0.0:1786177135.096927:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177135.096928:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177135.096930:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1786177135.096934:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177135.096936:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177135.096938:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5400.
00000100:00000001:3.0:1786177135.096938:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177135.096942:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177135.096943:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177135.096945:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.096950:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:3.0:1786177135.096951:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000400:00000010:0.0:1786177135.096955:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c0b0.
00000100:00000001:3.0:1786177135.096956:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177135.096965:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4088.
00000100:00100000:3.0:1786177135.096965:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1786177135.096973:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856340992, portal 25
00000100:00000001:3.0:1786177135.096974:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177135.096978:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177135.096979:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177135.096981:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
00000100:00000040:3.0:1786177135.096982:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014f55bbb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177135.096990:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177135.096993:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177135.096998:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177135.097002:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1786177135.097003:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000040:0.0:1786177135.097007:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177135.097008:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:0.0:1786177135.097010:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4550.
00000100:00080000:3.0:1786177135.097011:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:0.0:1786177135.097015:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856340992, offset 0
00000100:00000001:3.0:1786177135.097016:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177135.097021:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7358
00000400:00000200:0.0:1786177135.097025:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177135.097029:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177135.097033:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177135.097035:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786177135.097038:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1786177135.097041:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177135.097044:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786177135.097048:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:3.0:1786177135.097048:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177135.097053:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177135.097056:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:0.0:1786177135.097057:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:3.0:1786177135.097058:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177135.097061:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177135.097062:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786177135.097064:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177135.097065:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177135.097067:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177135.097070:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786177135.097073:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:3.0:1786177135.097073:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177135.097074:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177135.097077:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:0.0:1786177135.097079:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:3.0:1786177135.097082:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1786177135.097086:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1786177135.097087:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177135.097090:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786177135.097094:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254835:254835:256:4294967295] 192.168.202.110@tcp LPNI seq info [254824:254824:8:4294967295]
02000000:00000001:3.0:1786177135.097095:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177135.097099:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.097103:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177135.097103:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177135.097107:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177135.097109:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.097110:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177135.097115:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ca80.
00000400:00000200:0.0:1786177135.097116:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:3.0:1786177135.097119:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177135.097122:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177135.097123:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177135.097125:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177135.097126:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177135.097130:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177135.097131:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1786177135.097133:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45700.
02000000:00000010:3.0:1786177135.097135:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:3.0:1786177135.097139:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177135.097141:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:3.0:1786177135.097146:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1786177135.097151:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177135.097159:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177135.097161:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177135.097164:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ffb0865] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177135.097169:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.097170:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.097174:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177135.097177:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302034637
00000100:00080000:3.0:1786177135.097180:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:0.0:1786177135.097183:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00080000:3.0:1786177135.097184:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177135.097188:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177135.097190:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177135.097191:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177135.097192:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177135.097198:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7358
00000100:00000001:0.0:1786177135.097200:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302034637
00000400:00000200:3.0:1786177135.097203:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:0.0:1786177135.097206:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134de0.
00000400:00000200:3.0:1786177135.097209:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1786177135.097211:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00080000:3.0:1786177135.097212:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177135.097216:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1786177135.097216:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177135.097221:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.097222:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177135.097227:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177135.097227:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177135.097231:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177135.097231:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177135.097236:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177135.097237:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177135.097239:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786177135.097240:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000020:00000040:3.0:1786177135.097242:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:0.0:1786177135.097243:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1786177135.097249:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1786177135.097249:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:0.0:1786177135.097251:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177135.097255:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1786177135.097255:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177135.097261:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177135.097264:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177135.097271:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177135.097272:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177135.097273:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177135.097275:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177135.097276:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177135.097278:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177135.097281:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856341120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177135.097287:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:3.0:1786177135.097287:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:0.0:1786177135.097289:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177135.097292:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.097293:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:0.0:1786177135.097294:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:3.0:1786177135.097296:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
02000000:00000010:0.0:1786177135.097299:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
00000100:00080000:3.0:1786177135.097300:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1786177135.097303:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.097304:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.097308:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786177135.097308:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:0.0:1786177135.097313:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c738.
02000000:00000001:3.0:1786177135.097313:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177135.097319:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177135.097320:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c44c8.
00000100:00000001:3.0:1786177135.097321:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000200:0.0:1786177135.097324:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856341120, portal 10
00000008:00100000:3.0:1786177135.097325:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786177135.097327:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000008:00000001:3.0:1786177135.097329:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177135.097330:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000001:3.0:1786177135.097332:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177135.097337:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f480.
00000100:00000040:0.0:1786177135.097341:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177135.097342:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177135.097347:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177135.097350:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.097354:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786177135.097355:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:3.0:1786177135.097356:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:0.0:1786177135.097360:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786177135.097360:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177135.097380:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c47f8.
02000000:00000010:3.0:1786177135.097382:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000200:0.0:1786177135.097384:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856341120, offset 0
00000100:00000001:3.0:1786177135.097387:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.097392:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00100000:3.0:1786177135.097395:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1786177135.097400:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177135.097407:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1786177135.097410:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1786177135.097412:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1786177135.097414:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.097417:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:3.0:1786177135.097421:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092aa1aa7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177135.097422:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177135.097427:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177135.097433:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254836:254836:255:4294967295] 192.168.202.110@tcp LPNI seq info [254825:254825:7:4294967295]
00000400:00000200:0.0:1786177135.097443:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177135.097444:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.097449:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177135.097455:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786177135.097457:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:0.0:1786177135.097459:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45100.
00000100:00080000:3.0:1786177135.097460:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177135.097464:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:0.0:1786177135.097467:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786177135.097467:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177135.097472:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:0.1:1786177135.097475:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:3.0:1786177135.097477:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7358
00000100:00000001:0.0:1786177135.097479:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177135.097483:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177135.097485:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177135.097488:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:0.0:1786177135.097492:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1786177135.097493:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177135.097496:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302030857
00000100:00000040:3.0:1786177135.097499:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177135.097500:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302034637
00000100:00000001:3.0:1786177135.097503:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177135.097504:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302030857
00000100:00000040:3.0:1786177135.097507:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177135.097508:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302034637
00000100:00000001:0.0:1786177135.097530:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177135.097532:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177135.097535:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1786177135.097540:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:0.0:1786177135.097543:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177135.097544:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177135.097547:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177135.097551:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177135.097557:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:0.0:1786177135.097559:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177135.097562:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177135.097562:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177135.097569:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
02000000:00000001:0.0:1786177135.097570:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177135.097575:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1786177135.097578:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:0.0:1786177135.097580:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.097583:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:0.0:1786177135.097588:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856341248:192.168.202.110@tcp:38:kworker.0
00000100:00000040:3.0:1786177135.097590:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177135.097596:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1786177135.097597:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:0.0:1786177135.097599:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177135.097603:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177135.097604:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
02000000:00000001:0.0:1786177135.097605:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177135.097610:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7400.
00000100:00080000:3.0:1786177135.097611:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1786177135.097613:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.097619:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177135.097619:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177135.097623:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c3c8.
02000000:00000001:3.0:1786177135.097624:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:0.0:1786177135.097629:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4cc0.
02000000:00000001:3.0:1786177135.097631:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:0.0:1786177135.097636:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856341248, portal 10
02000000:00000001:3.0:1786177135.097636:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.097638:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177135.097641:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000008:00100000:3.0:1786177135.097642:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786177135.097645:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000008:00000001:3.0:1786177135.097647:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.097652:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177135.097655:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1786177135.097659:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f100.
00000020:00000040:3.0:1786177135.097665:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:0.0:1786177135.097671:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:3.0:1786177135.097672:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:0.0:1786177135.097677:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786177135.097677:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177135.097682:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:0.0:1786177135.097683:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4110.
02000000:00000001:3.0:1786177135.097685:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:0.0:1786177135.097688:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856341248, offset 0
02000000:00000001:3.0:1786177135.097690:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.097697:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000010:3.0:1786177135.097698:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000400:00000200:0.0:1786177135.097704:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177135.097705:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.097711:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:3.0:1786177135.097715:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1786177135.097718:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177135.097725:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177135.097730:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177135.097735:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1786177135.097747:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177135.097749:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.097753:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254837:254837:254:4294967295] 192.168.202.110@tcp LPNI seq info [254826:254826:6:4294967295]
00000100:00000040:3.0:1786177135.097759:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1ef9f45] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177135.097770:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177135.097778:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.097785:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1786177135.097786:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7368)
00000800:00000200:0.0:1786177135.097794:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177135.097799:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45200.
00000800:00000200:0.0:1786177135.097805:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177135.097809:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177135.097812:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177135.097816:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177135.097819:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177135.097821:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302030983
00000100:00000001:0.0:1786177135.097823:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302034637
00000100:00000001:0.0:1786177135.097825:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302030983
00000100:00000001:0.0:1786177135.097827:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302034637
00000100:00000001:0.0:1786177135.097829:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177135.097832:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177135.097835:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177135.097837:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177135.097840:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177135.097842:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1786177135.097844:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177135.097846:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177135.097847:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177135.097851:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177135.097857:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177135.097859:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177135.097861:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177135.097863:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177135.097865:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177135.097868:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856341376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177135.097871:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177135.097873:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177135.097874:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177135.097876:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177135.097879:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786177135.097882:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.097885:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177135.097888:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381caa8.
00000400:00000010:0.0:1786177135.097892:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177135.097895:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856341376, portal 4
00000100:00000001:0.0:1786177135.097897:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177135.097898:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:0.0:1786177135.097905:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177135.097912:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177135.097915:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177135.097918:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4a18.
00000100:00000200:0.0:1786177135.097921:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856341376, offset 0
00000400:00000200:0.0:1786177135.097926:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177135.097930:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177135.097933:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177135.097936:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177135.097939:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177135.097942:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177135.097944:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177135.097948:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254838:254838:253:4294967295] 192.168.202.110@tcp LPNI seq info [254827:254827:5:4294967295]
00000400:00000200:0.0:1786177135.097952:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177135.097959:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177135.097964:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177135.097968:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45a00.
00000800:00000200:0.0:1786177135.097972:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177135.097975:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177135.097978:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177135.097981:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177135.097984:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177135.097986:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302030858
00000100:00000001:0.0:1786177135.097989:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302034637
00000100:00000001:0.0:1786177135.097991:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302030858
00000100:00000001:0.0:1786177135.097993:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302034637
00000100:00000001:0.0:1786177135.097995:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177135.097998:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177135.098000:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177135.098004:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177135.098012:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177135.098014:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177135.098016:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177135.098018:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177135.098020:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177135.098024:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856341504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177135.098027:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177135.098029:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177135.098031:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177135.098033:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177135.098036:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4800.
02000000:00000001:0.0:1786177135.098039:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177135.098042:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177135.098045:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177135.098048:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177135.098052:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856341504, portal 4
00000100:00000001:0.0:1786177135.098054:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177135.098055:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:0.0:1786177135.098061:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177135.098067:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177135.098069:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177135.098071:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4198.
00000100:00000200:0.0:1786177135.098073:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856341504, offset 0
00000400:00000200:0.0:1786177135.098077:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177135.098081:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177135.098085:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177135.098088:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177135.098090:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177135.098093:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177135.098095:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177135.098098:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254839:254839:252:4294967295] 192.168.202.110@tcp LPNI seq info [254828:254828:4:4294967295]
00000400:00000200:0.0:1786177135.098102:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177135.098108:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177135.098112:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177135.098116:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177135.098120:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177135.098124:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177135.098126:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177135.098130:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177135.098133:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177135.098135:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302030866
00000100:00000001:0.0:1786177135.098137:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302034637
00000100:00000001:0.0:1786177135.098139:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302030866
00000100:00000001:0.0:1786177135.098141:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302034637
00000100:00000001:0.0:1786177135.098144:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177135.098146:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177135.098148:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177135.098151:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177135.098154:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177135.098156:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1786177135.098158:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177135.098159:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177135.098161:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177135.098163:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177135.672628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177135.672634:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177135.672638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177135.672639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177135.672641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177135.672642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177135.672691:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177135.672695:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177135.672698:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177136.120618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177136.120636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177136.120639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177136.120646:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177136.120650:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177136.120653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177136.120656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177136.120659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177136.120662:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177136.696621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177136.696626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177136.696630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177136.696631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177136.696634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177136.696634:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177136.696710:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177136.696715:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177136.696718:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177137.144677:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177137.144684:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177137.144686:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177137.144690:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177137.144694:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177137.144698:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177137.144700:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177137.144703:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177137.144706:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177137.720635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177137.720643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177137.720649:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177137.720651:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177137.720654:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177137.720656:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177137.720709:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177137.720713:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177137.720715:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177138.168735:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177138.168741:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177138.168742:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177138.168746:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177138.168749:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177138.168752:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177138.168754:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177138.168756:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177138.168758:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:3.0:1786177138.168896:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:3.0:1786177138.168956:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:3.0:1786177138.168968:0:2382:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177138.168973:0:2382:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:3.0:1786177138.168980:0:2382:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177138.168986:0:2382:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:3.0:1786177138.168992:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:3.0:1786177138.169001:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45700.
00000400:00000200:3.0:1786177138.169009:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1786177138.169015:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1786177138.169024:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4550
00000400:00000010:3.0:1786177138.169027:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4550.
00000100:00000001:3.0:1786177138.169034:0:2382:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177138.169045:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177138.169079:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177138.169085:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177138.169095:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177138.169098:0:2382:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:3.0:1786177138.169112:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:3.0:1786177138.169115:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45100.
00000400:00000200:3.0:1786177138.169118:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1786177138.169123:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1786177138.169128:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c47f8
00000400:00000010:3.0:1786177138.169131:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c47f8.
00000100:00000001:3.0:1786177138.169134:0:2382:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177138.169139:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177138.169150:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177138.169152:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1786177138.169154:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.169161:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177138.169167:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177138.169170:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1786177138.169174:0:2382:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:0.0:1786177138.169183:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177135/real 1786177138]  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:3.0:1786177138.169184:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:3.0:1786177138.169189:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45200.
00000400:00000200:3.0:1786177138.169195:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1786177138.169203:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1786177138.169209:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4110
00000400:00000010:3.0:1786177138.169213:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4110.
00000100:00000001:3.0:1786177138.169217:0:2382:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177138.169224:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177138.169227:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177138.169232:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1786177138.169237:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000400:00000010:0.0:1786177138.169239:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c738.
00000100:00000040:3.0:1786177138.169242:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177138.169248:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c44c8
00000400:00000010:0.0:1786177138.169253:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c44c8.
00000100:00000001:3.0:1786177138.169254:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177138.169259:0:2382:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177138.169261:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000800:00000100:3.0:1786177138.169266:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:3.0:1786177138.169270:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45a00.
00000100:00000200:0.0:1786177138.169272:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177138.169274:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1786177138.169283:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1786177138.169290:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4a18
00000400:00000010:3.0:1786177138.169294:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4a18.
00000100:00000001:3.0:1786177138.169298:0:2382:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177138.169299:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177138.169312:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.169313:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177138.169317:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.169324:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177138.169328:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177138.169328:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00100000:0.0:1786177138.169354:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.169364:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177138.169371:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177138.169375:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177138.169393:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177138.169397:0:2382:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177138.169398:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177138.169403:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.169407:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000800:00000100:3.0:1786177138.169408:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000100:00000001:0.0:1786177138.169410:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000800:00000010:3.0:1786177138.169413:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000100:00000001:0.0:1786177138.169417:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000400:00000200:3.0:1786177138.169419:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:0.0:1786177138.169424:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000400:00000200:3.0:1786177138.169425:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1786177138.169430:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4198
00000100:00080000:0.0:1786177138.169434:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000400:00000010:3.0:1786177138.169434:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4198.
00000100:00000001:3.0:1786177138.169439:0:2382:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177138.169448:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177138.169452:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177138.169462:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1786177138.169464:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177138.169469:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177160 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177138.169476:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177138.169482:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177138.169486:0:2382:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:0.0:1786177138.169496:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856341120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177138.169504:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177138.169508:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177135/real 1786177138]  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177138.169575:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177138.169577:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177138.169580:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c3c8.
00000400:00000200:0.0:1786177138.169583:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4cc0
00000400:00000010:0.0:1786177138.169585:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4cc0.
00000100:00000001:0.0:1786177138.169588:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177138.169596:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177138.169607:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.169615:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177138.169617:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.169620:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177138.169622:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177138.169627:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.169634:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177138.169638:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.169644:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177138.169646:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.169648:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177138.169650:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.169651:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177138.169654:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177138.169656:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177138.169661:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177138.169664:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177138.169669:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177138.169678:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856341248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177138.169683:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177138.169687:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177135/real 1786177138]  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177138.169697:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177138.169699:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177138.169702:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c0b0.
00000400:00000200:0.0:1786177138.169705:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4088
00000400:00000010:0.0:1786177138.169708:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4088.
00000100:00000001:0.0:1786177138.169710:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177138.169714:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177138.169722:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.169729:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177138.169731:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.169732:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177138.169734:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177138.169738:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.169755:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177138.169758:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.169765:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177138.169768:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.169770:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177138.169771:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.169773:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177138.169775:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177138.169778:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177138.169783:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177138.169785:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177138.169789:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177138.169798:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856340992:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177138.169803:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177138.169812:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177138.169816:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856341120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.169825:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177138.169830:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177138.169836:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786177138.169842:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177138.169846:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177138.169849:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177138.169851:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177138.169857:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786177138.169863:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177138.169866:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177138.169869:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:0.0:1786177138.169874:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177138.169876:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.169878:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177138.169882:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856341248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.169888:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177138.169890:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177138.169892:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7400.
02000000:00000001:0.0:1786177138.169894:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177138.169895:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177138.169897:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177138.169899:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177138.169901:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786177138.169903:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177138.169904:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177138.169905:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:0.0:1786177138.169908:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177138.169910:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.169912:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177138.169915:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856340992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.169923:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177138.169925:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177138.169927:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177138.169930:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177138.169931:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177138.169933:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177138.169935:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177138.169936:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:0.0:1786177138.169938:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177138.169940:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177138.169942:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000001:0.0:1786177138.169943:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177138.169945:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.169952:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177138.169955:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177138.169957:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177138.169958:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177138.169962:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177135/real 1786177138]  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177138.169970:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177138.169973:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177138.169975:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381caa8.
00000400:00000200:0.0:1786177138.169978:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177138.169980:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177138.169982:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177138.169986:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177138.169994:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.170002:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177138.170004:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.170005:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177138.170007:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177138.170011:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.170025:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177138.170029:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.170036:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177138.170038:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.170040:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177138.170041:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.170043:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177138.170045:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177138.170048:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177138.170052:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177138.170054:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177138.170058:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177138.170066:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856341376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177138.170070:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177138.170074:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177135/real 1786177138]  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177138.170081:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177138.170083:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177138.170085:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177138.170088:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177138.170090:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177138.170092:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177138.170095:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177138.170110:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.170118:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177138.170119:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.170121:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177138.170123:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177138.170127:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.170134:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177138.170138:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.170153:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177138.170156:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.170157:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177138.170159:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.170161:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177138.170163:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177138.170166:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177138.170170:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177138.170172:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177138.170177:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177138.170186:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856341504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177138.170191:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.170193:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177138.170196:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856341376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.170203:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177138.170204:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177138.170205:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786177138.170208:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177138.170209:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177138.170210:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177138.170212:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177138.170213:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786177138.170215:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177138.170217:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177138.170218:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:0.0:1786177138.170221:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177138.170223:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.170226:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177138.170229:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856341504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177160 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177138.170236:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177138.170237:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177138.170239:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4800.
02000000:00000001:0.0:1786177138.170242:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177138.170243:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177138.170245:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177138.170247:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177138.170248:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786177138.170251:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177138.170253:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177138.170254:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:0.0:1786177138.170256:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177138.170258:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.170260:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177138.170270:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177138.170273:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177138.170275:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177138.170277:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177138.744655:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177138.744662:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177138.744667:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177138.744668:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177138.744671:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177138.744672:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177138.744785:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177138.744791:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177138.744795:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177138.936603:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302034637, at 4302034688
00000100:00000040:1.0:1786177138.936603:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302034637, at 4302034688
00000100:00000010:0.0:1786177138.936622:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000010:1.0:1786177138.936627:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134de0.
00000100:00000001:2.0:1786177139.192680:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177139.192686:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177139.192687:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177139.192692:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177139.192694:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177139.192698:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177139.192700:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177139.192702:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177139.192704:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177139.768591:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177139.768596:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177139.768600:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177139.768601:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177139.768603:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177139.768604:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177139.768656:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177139.768659:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177139.768662:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786177140.216658:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786177140.216658:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177140.216666:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177140.216668:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177140.216677:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:2.0:1786177140.216678:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177140.216686:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177140.216687:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1786177140.216690:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177140.216694:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177140.216695:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177140.216698:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177140.216702:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.216706:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177140.216717:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7363
00000400:00000200:0.0:1786177140.216729:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177140.216739:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177140.216744:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177140.216749:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.216753:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177140.216757:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.216760:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177140.216765:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177140.216768:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177140.216771:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177140.216775:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177140.216778:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177140.216782:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177140.216786:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.216788:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177140.216797:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.216800:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177140.216804:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177140.216808:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177140.216812:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177140.216816:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177140.216820:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177140.216824:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.216827:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177140.216831:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177140.216842:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418380.
00000020:00000040:0.0:1786177140.216846:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177140.216851:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177140.216854:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.216858:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177140.216862:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177140.216868:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177140.216879:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
00000100:00000001:0.0:1786177140.216888:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177140.216911:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177140.216924:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177140.216930:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177140.216937:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e6b901d7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177140.216977:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177140.216981:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177140.216985:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177140.216986:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177140.216988:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177140.216992:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177140.216995:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7363
00000400:00000200:0.0:1786177140.217000:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177140.217003:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177140.217005:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177140.217009:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217011:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177140.217013:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217015:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177140.217017:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177140.217019:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177140.217020:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177140.217022:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177140.217025:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177140.217027:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177140.217031:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217033:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177140.217035:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217038:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177140.217041:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177140.217044:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217047:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177140.217049:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177140.217057:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177140.217059:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.217062:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177140.217067:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177140.217071:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.217073:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177140.217077:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419f80.
00000020:00000040:0.0:1786177140.217080:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177140.217083:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177140.217085:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.217087:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177140.217089:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177140.217091:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177140.217096:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000100:00000001:0.0:1786177140.217100:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177140.217106:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177140.217113:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177140.217116:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177140.217119:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ad753ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177140.217128:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177140.217131:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177140.217135:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177140.217137:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177140.217140:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177140.217144:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177140.217146:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7363
00000400:00000200:0.0:1786177140.217150:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177140.217154:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177140.217163:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177140.217166:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217168:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177140.217171:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217173:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177140.217176:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177140.217178:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177140.217180:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177140.217182:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177140.217185:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177140.217187:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177140.217191:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217194:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177140.217196:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217198:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177140.217201:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786177140.217204:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217208:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177140.217209:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177140.217213:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177140.217215:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.217218:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177140.217220:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177140.217223:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.217224:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177140.217231:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b800.
00000020:00000040:0.0:1786177140.217234:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177140.217237:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177140.217238:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.217240:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177140.217241:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177140.217244:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177140.217248:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000100:00000001:0.0:1786177140.217252:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177140.217257:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177140.217265:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177140.217268:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177140.217272:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000192ee9a0] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177140.217282:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177140.217285:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177140.217289:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177140.217290:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177140.217292:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177140.217296:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177140.217299:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7363
00000400:00000200:0.0:1786177140.217303:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177140.217306:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177140.217310:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177140.217330:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217332:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177140.217334:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217336:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177140.217339:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177140.217340:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177140.217342:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177140.217345:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177140.217347:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177140.217349:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177140.217353:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217355:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177140.217357:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217360:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177140.217362:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177140.217365:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217368:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177140.217370:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177140.217373:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177140.217375:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.217377:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177140.217380:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177140.217383:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.217385:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177140.217389:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418000.
00000020:00000040:0.0:1786177140.217392:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177140.217395:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177140.217396:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.217398:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177140.217400:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177140.217402:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177140.217407:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6000.
00000100:00000001:0.0:1786177140.217410:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177140.217416:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177140.217425:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177140.217427:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177140.217430:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002e10e4f5] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177140.217446:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177140.217449:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177140.217452:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177140.217453:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177140.217455:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177140.217457:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177140.217459:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7363
00000400:00000200:0.0:1786177140.217463:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177140.217466:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177140.217468:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177140.217471:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217473:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177140.217475:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217477:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177140.217479:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177140.217480:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177140.217482:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177140.217484:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177140.217487:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:0.0:1786177140.217489:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786177140.217492:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217495:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177140.217497:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217498:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177140.217501:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177140.217504:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177140.217507:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177140.217561:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177140.217564:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177140.217567:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.217569:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177140.217571:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177140.217573:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.217575:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177140.217579:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418700.
00000020:00000040:0.0:1786177140.217582:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177140.217585:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177140.217586:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177140.217588:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177140.217590:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177140.217592:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177140.217596:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:0.0:1786177140.217600:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177140.217605:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177140.217612:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177140.217614:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177140.217618:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bf66f89d] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177140.217625:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177140.217632:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7373)
00000100:00000001:3.0:1786177140.217822:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177140.217828:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177140.217831:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177140.217840:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.217861:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177140.217863:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177140.217868:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177140.217872:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177140.217875:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177140.217880:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856341632:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177140.217886:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177140.217892:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177140.217895:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177140.217899:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177140.217905:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177140.217908:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177140.217914:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177140.217919:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177140.217929:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177140.217938:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856341632, portal 25
00000100:00000001:3.0:1786177140.217941:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177140.217942:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761655680 : -114547947895936 : ffff97d1ba418380)
00000100:00000040:3.0:1786177140.217948:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.217957:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177140.217961:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177140.217963:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177140.217968:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856341632, offset 0
00000400:00000200:3.0:1786177140.217976:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177140.217986:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177140.217994:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177140.218002:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177140.218008:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177140.218012:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177140.218016:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177140.218020:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254840:254840:256:4294967295] 192.168.202.110@tcp LPNI seq info [254829:254829:8:4294967295]
00000400:00000200:3.0:1786177140.218025:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177140.218037:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177140.218045:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177140.218052:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177140.218060:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177140.218065:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177140.218071:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177140.218075:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177140.218081:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177140.218084:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177140.218089:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177140.218095:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177140.218103:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177140.218106:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177140.218112:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177140.218116:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218127:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177140.218130:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218139:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177140.218141:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177140.218150:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177140.218155:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177140.218159:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302039758
00000100:00000010:3.0:1786177140.218165:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37960.
00000100:00000001:3.0:1786177140.218174:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177140.218182:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302039758
00000100:00000010:3.0:1786177140.218185:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37480.
00000100:00000001:3.0:1786177140.218188:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177140.218193:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177140.218196:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177140.218199:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177140.218205:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177140/real 1786177140]  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177140.218216:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177140.218218:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177140.218221:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177140.218224:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177140.218225:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177140.218229:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177140.218232:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177140.218241:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218247:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177140.218249:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.218252:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177140.218254:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177140.218258:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218265:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177140.218270:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218277:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177140.218280:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.218282:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177140.218283:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.218286:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177140.218288:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177140.218292:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177140.218297:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177140.218299:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177140.218304:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177140.218329:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856341632:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177140.218339:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177140.218344:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218350:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177140.218352:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177140.218354:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177140.218356:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177140.218359:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177140.218362:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856341760:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177140.218366:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177140.218373:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177140.218375:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177140.218377:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177140.218382:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177140.218384:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177140.218388:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177140.218391:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177140.218395:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177140.218398:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856341760, portal 10
00000100:00000001:3.0:1786177140.218401:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177140.218402:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761662848 : -114547947888768 : ffff97d1ba419f80)
00000100:00000040:3.0:1786177140.218410:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218418:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177140.218420:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177140.218423:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177140.218426:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856341760, offset 0
00000400:00000200:3.0:1786177140.218431:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177140.218435:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177140.218439:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177140.218442:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177140.218446:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177140.218448:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177140.218451:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177140.218454:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254841:254841:256:4294967295] 192.168.202.110@tcp LPNI seq info [254830:254830:8:4294967295]
00000400:00000200:3.0:1786177140.218458:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177140.218465:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177140.218470:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177140.218474:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177140.218478:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177140.218481:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177140.218486:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177140.218489:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177140.218493:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177140.218494:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177140.218498:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177140.218501:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177140.218504:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177140.218506:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177140.218508:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177140.218542:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218549:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177140.218552:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218559:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177140.218560:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177140.218565:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177140.218570:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177140.218573:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177140.218574:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302035979
00000100:00000001:3.0:1786177140.218577:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302039758
00000100:00000001:3.0:1786177140.218580:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302035979
00000100:00000001:3.0:1786177140.218583:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302039758
00000100:00000001:3.0:1786177140.218585:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177140.218587:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177140.218589:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177140.218593:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177140/real 1786177140]  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177140.218601:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177140.218603:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177140.218604:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177140.218607:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177140.218608:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177140.218611:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177140.218614:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177140.218622:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218630:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177140.218631:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.218633:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177140.218635:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177140.218645:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218653:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177140.218658:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218665:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177140.218667:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.218669:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177140.218670:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.218673:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177140.218676:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177140.218678:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177140.218683:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177140.218685:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177140.218690:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177140.218699:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856341760:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177140.218703:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177140.218707:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218714:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177140.218716:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177140.218718:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177140.218720:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177140.218722:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177140.218725:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856341888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177140.218728:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177140.218731:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177140.218732:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177140.218735:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177140.218739:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177140.218742:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177140.218745:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177140.218748:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177140.218752:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177140.218756:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856341888, portal 10
00000100:00000001:3.0:1786177140.218758:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177140.218759:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761669120 : -114547947882496 : ffff97d1ba41b800)
00000100:00000040:3.0:1786177140.218766:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218773:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177140.218780:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177140.218783:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177140.218786:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856341888, offset 0
00000400:00000200:3.0:1786177140.218790:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177140.218799:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177140.218803:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177140.218807:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177140.218809:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177140.218811:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177140.218814:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177140.218819:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254842:254842:256:4294967295] 192.168.202.110@tcp LPNI seq info [254831:254831:8:4294967295]
00000400:00000200:3.0:1786177140.218824:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177140.218832:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177140.218837:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177140.218841:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177140.218846:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177140.218850:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177140.218854:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177140.218858:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177140.218862:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177140.218864:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177140.218867:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177140.218870:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177140.218872:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177140.218874:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177140.218876:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177140.218878:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218884:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177140.218886:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218896:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177140.218897:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177140.218900:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177140.218903:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177140.218907:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177140.218909:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302036104
00000100:00000001:3.0:1786177140.218912:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302039758
00000100:00000001:3.0:1786177140.218914:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302036104
00000100:00000001:3.0:1786177140.218917:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302039758
00000100:00000001:3.0:1786177140.218919:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177140.218922:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177140.218924:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177140.218928:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177140/real 1786177140]  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177140.218937:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177140.218939:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177140.218940:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177140.218943:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177140.218945:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177140.218947:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177140.218951:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177140.218957:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218962:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177140.218963:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.218964:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177140.218965:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177140.218967:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218972:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177140.218975:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.218980:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177140.218981:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.218982:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177140.218983:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.218985:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177140.218987:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177140.218989:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177140.218993:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177140.218995:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177140.218998:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177140.219006:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856341888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177140.219011:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177140.219015:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219022:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177140.219023:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177140.219026:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177140.219028:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177140.219030:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177140.219033:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856342016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177140.219037:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177140.219038:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177140.219041:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177140.219042:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177140.219046:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177140.219048:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177140.219052:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177140.219054:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177140.219058:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177140.219061:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856342016, portal 4
00000100:00000001:3.0:1786177140.219063:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177140.219065:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761654784 : -114547947896832 : ffff97d1ba418000)
00000100:00000040:3.0:1786177140.219071:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219077:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177140.219080:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177140.219082:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177140.219085:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856342016, offset 0
00000400:00000200:3.0:1786177140.219089:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177140.219094:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177140.219097:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177140.219101:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177140.219103:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177140.219106:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177140.219109:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177140.219112:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254843:254843:256:4294967295] 192.168.202.110@tcp LPNI seq info [254832:254832:8:4294967295]
00000400:00000200:3.0:1786177140.219116:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177140.219124:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177140.219129:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177140.219133:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177140.219137:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177140.219140:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177140.219145:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177140.219148:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177140.219152:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177140.219155:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177140.219158:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177140.219161:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177140.219163:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177140.219164:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177140.219166:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177140.219169:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219175:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177140.219177:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219182:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177140.219183:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177140.219185:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177140.219188:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177140.219190:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177140.219192:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302035979
00000100:00000001:3.0:1786177140.219194:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302039758
00000100:00000001:3.0:1786177140.219196:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302035979
00000100:00000001:3.0:1786177140.219197:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302039758
00000100:00000001:3.0:1786177140.219198:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177140.219200:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177140.219201:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177140.219205:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177140/real 1786177140]  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177140.219213:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177140.219214:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177140.219216:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177140.219219:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177140.219220:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177140.219223:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177140.219227:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177140.219236:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219243:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177140.219245:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219247:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177140.219248:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177140.219252:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219259:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177140.219263:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219270:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177140.219273:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219275:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177140.219276:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219278:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177140.219279:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177140.219282:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177140.219285:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177140.219287:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177140.219291:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177140.219298:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856342016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177140.219303:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177140.219306:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219330:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177140.219332:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177140.219336:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177140.219337:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177140.219340:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177140.219343:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856342144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177140.219347:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177140.219349:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177140.219351:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177140.219354:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177140.219357:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177140.219359:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177140.219362:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177140.219365:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177140.219368:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177140.219371:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856342144, portal 4
00000100:00000001:3.0:1786177140.219373:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177140.219375:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761656576 : -114547947895040 : ffff97d1ba418700)
00000100:00000040:3.0:1786177140.219380:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219386:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177140.219389:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177140.219391:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177140.219393:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856342144, offset 0
00000400:00000200:3.0:1786177140.219397:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177140.219401:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177140.219405:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177140.219408:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177140.219410:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177140.219413:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177140.219415:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177140.219418:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254844:254844:256:4294967295] 192.168.202.110@tcp LPNI seq info [254833:254833:8:4294967295]
00000400:00000200:3.0:1786177140.219423:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177140.219429:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177140.219433:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177140.219436:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177140.219440:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177140.219442:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177140.219446:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177140.219448:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177140.219452:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177140.219454:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177140.219457:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177140.219460:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177140.219463:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177140.219465:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177140.219467:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177140.219470:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219478:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177140.219481:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177165 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219487:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177140.219488:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177140.219491:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177140.219494:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177140.219497:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177140.219499:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302035987
00000100:00000001:3.0:1786177140.219502:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302039758
00000100:00000001:3.0:1786177140.219504:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302035987
00000100:00000001:3.0:1786177140.219506:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302039758
00000100:00000001:3.0:1786177140.219509:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177140.219536:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177140.219538:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177140.219542:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177140/real 1786177140]  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177140.219550:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177140.219551:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177140.219553:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177140.219556:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177140.219558:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177140.219560:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177140.219563:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177140.219571:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219578:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177140.219579:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219581:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177140.219582:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177140.219585:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219592:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177140.219596:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219602:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177140.219605:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219606:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177140.219608:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219610:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177140.219612:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177140.219614:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177140.219619:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177140.219621:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177140.219625:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177140.219633:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856342144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177140.219637:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219639:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177140.219643:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418380 x1872934856341632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219650:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177140.219653:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177140.219657:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177140.219660:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177140.219663:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177140.219665:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177140.219668:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177140.219671:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
02000000:00000001:3.0:1786177140.219676:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177140.219678:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177140.219680:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418380.
00000100:00000001:3.0:1786177140.219685:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177140.219687:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219689:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177140.219692:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419f80 x1872934856341760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219700:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177140.219701:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177140.219703:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177140.219705:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177140.219706:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177140.219707:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177140.219709:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177140.219710:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:3.0:1786177140.219712:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177140.219713:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177140.219714:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419f80.
00000100:00000001:3.0:1786177140.219716:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177140.219717:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219718:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177140.219721:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b800 x1872934856341888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219725:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177140.219726:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177140.219727:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177140.219729:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177140.219730:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177140.219731:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177140.219732:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177140.219733:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:3.0:1786177140.219734:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177140.219735:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177140.219736:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b800.
00000100:00000001:3.0:1786177140.219739:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177140.219740:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219742:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177140.219745:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418000 x1872934856342016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219751:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177140.219752:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177140.219754:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177140.219756:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177140.219757:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177140.219759:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177140.219761:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177140.219762:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6000.
02000000:00000001:3.0:1786177140.219765:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177140.219766:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177140.219768:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418000.
00000100:00000001:3.0:1786177140.219771:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177140.219772:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219774:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177140.219777:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418700 x1872934856342144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177165 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177140.219783:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177140.219784:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177140.219786:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177140.219788:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177140.219789:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177140.219791:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177140.219793:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177140.219794:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:3.0:1786177140.219797:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177140.219798:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177140.219800:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418700.
00000100:00000001:3.0:1786177140.219802:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177140.219804:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219806:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177140.219815:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177140.219818:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177140.219820:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177140.219823:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177140.792667:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177140.792679:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177140.792682:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177140.792687:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177140.792693:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177140.792696:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177140.792698:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177140.792701:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177140.792706:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177140.792708:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177140.792709:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177140.792710:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177140.792712:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177140.792714:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177140.792716:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177140.792718:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177140.792722:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177140.792723:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177140.792794:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177140.792798:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177140.792802:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177141.240791:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177141.240799:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177141.240801:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177141.240806:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177141.240809:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177141.240813:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177141.240815:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177141.240818:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177141.240820:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177141.816611:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177141.816623:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177141.816630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177141.816631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177141.816633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177141.816635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177141.816691:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177141.816697:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177141.816701:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177142.264588:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177142.264597:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177142.264600:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177142.264606:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177142.264610:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177142.264613:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177142.264615:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177142.264618:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177142.264621:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177142.840667:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177142.840675:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177142.840680:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177142.840681:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177142.840684:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177142.840687:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177142.840689:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177142.840693:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177142.840696:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177142.840698:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177142.840700:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177142.840702:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177142.840783:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177142.840788:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177142.840790:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177143.288647:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177143.288658:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177143.288662:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177143.288670:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177143.288675:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177143.288680:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177143.288684:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177143.288687:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177143.288691:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177143.864624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177143.864630:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177143.864634:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177143.864636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177143.864638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177143.864639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177143.864740:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177143.864747:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177143.864752:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177144.056651:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302039758, at 4302039808
00000100:00000040:0.0:1786177144.056663:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302039758, at 4302039808
00000100:00000010:1.0:1786177144.056666:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37480.
00000100:00000010:0.0:1786177144.056682:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37960.
00000100:00000001:2.0:1786177144.312653:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177144.312662:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177144.312664:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177144.312670:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177144.312674:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177144.312678:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177144.312681:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177144.312684:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177144.312686:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177144.888593:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177144.888599:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177144.888602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177144.888603:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177144.888606:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177144.888607:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177144.888676:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177144.888681:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177144.888685:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.0:1786177145.208679:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786177145.208688:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786177145.208693:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:0.0:1786177145.208855:0:2384:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1786177145.208860:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786177145.208870:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1786177145.208875:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786177145.208878:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177145.336594:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177145.336601:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177145.336603:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177145.336608:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177145.336611:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1786177145.336611:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786177145.336616:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786177145.336617:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177145.336619:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177145.336622:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1786177145.336623:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177145.336627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177145.336628:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177145.336635:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177145.336643:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7368
00000400:00000200:3.0:1786177145.336653:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177145.336660:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177145.336664:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177145.336667:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.336671:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177145.336673:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.336675:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177145.336678:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177145.336680:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177145.336682:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177145.336685:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177145.336694:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177145.336695:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177145.336697:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.336699:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177145.336700:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.336701:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177145.336704:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177145.336706:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177145.336707:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177145.336710:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177145.336712:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177145.336714:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177145.336715:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177145.336717:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177145.336725:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d500.
00000020:00000040:3.0:1786177145.336727:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177145.336731:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177145.336732:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177145.336735:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177145.336738:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177145.336741:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177145.336747:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786177145.336752:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177145.336759:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177145.336767:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177145.336771:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177145.336775:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7c36393] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177145.336793:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177145.336796:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177145.336798:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177145.336799:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177145.336800:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177145.336802:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177145.336804:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7368
00000400:00000200:3.0:1786177145.336808:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177145.336810:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177145.336812:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177145.336815:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.336817:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177145.336819:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1786177145.336819:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177145.336824:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177145.336824:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177145.336826:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:0.0:1786177145.336828:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1786177145.336828:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177145.336831:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177145.336833:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786177145.336836:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177145.336837:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177145.336838:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177145.336844:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.336848:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177145.336853:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177145.336854:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177145.336855:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177145.336857:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177145.336859:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
02000000:00000001:0.0:1786177145.336861:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:3.0:1786177145.336863:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1786177145.336865:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177145.336870:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177145.336871:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177145.336875:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:0.0:1786177145.336878:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856342272:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1786177145.336881:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177145.336882:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177145.336886:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177145.336887:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1786177145.336888:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:0.0:1786177145.336890:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177145.336893:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:3.0:1786177145.336893:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177145.336897:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177145.336899:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5800.
00000100:00000001:3.0:1786177145.336899:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177145.336902:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177145.336903:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ea00.
00000020:00000040:3.0:1786177145.336906:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000400:00000200:0.0:1786177145.336910:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786177145.336911:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177145.336913:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177145.336915:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000100:00000001:3.0:1786177145.336915:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177145.336918:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177145.336920:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177145.336923:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
02000000:00000010:3.0:1786177145.336925:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:3.0:1786177145.336929:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786177145.336931:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856342272, portal 25
00000100:00100000:3.0:1786177145.336935:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177145.336936:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177145.336940:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000100:00000001:3.0:1786177145.336947:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177145.336949:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177145.336953:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177145.336954:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053162887] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177145.336982:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.336984:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1786177145.336985:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177145.336989:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000040:0.0:1786177145.336990:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177145.336991:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177145.336993:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:0.0:1786177145.336995:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000001:3.0:1786177145.336995:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:0.0:1786177145.337000:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856342272, offset 0
00000100:00080000:3.0:1786177145.337000:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7368
00000400:00000200:3.0:1786177145.337005:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177145.337009:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:0.0:1786177145.337014:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786177145.337014:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177145.337019:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.337021:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177145.337022:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.337024:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786177145.337028:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1786177145.337028:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177145.337032:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177145.337034:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177145.337036:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:0.0:1786177145.337038:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:3.0:1786177145.337039:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177145.337043:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:0.0:1786177145.337045:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1786177145.337048:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.337050:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786177145.337052:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:3.0:1786177145.337055:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.337057:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177145.337059:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786177145.337061:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177145.337063:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177145.337064:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786177145.337066:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1786177145.337066:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177145.337069:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177145.337072:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.337073:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177145.337076:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1786177145.337080:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254845:254845:256:4294967295] 192.168.202.110@tcp LPNI seq info [254834:254834:8:4294967295]
00000008:00000001:3.0:1786177145.337080:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177145.337083:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177145.337088:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:3.0:1786177145.337089:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c000.
00000020:00000040:3.0:1786177145.337093:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177145.337095:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177145.337098:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177145.337102:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1786177145.337103:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:3.0:1786177145.337105:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177145.337108:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177145.337111:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000010:3.0:1786177145.337111:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000001:3.0:1786177145.337117:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1786177145.337119:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000100:00100000:3.0:1786177145.337121:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.0:1786177145.337129:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177145.337133:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000200:0.0:1786177145.337135:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:3.0:1786177145.337136:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177145.337144:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006fe8fdb7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.1:1786177145.337145:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177145.337151:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:3.0:1786177145.337156:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177145.337159:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000100:0.0:1786177145.337161:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786177145.337162:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177145.337166:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000010:0.0:1786177145.337168:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000100:00080000:3.0:1786177145.337168:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177145.337173:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177145.337176:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7368
00000400:00000200:0.0:1786177145.337178:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177145.337179:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177145.337184:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177145.337186:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786177145.337190:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:3.0:1786177145.337191:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.337196:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177145.337198:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177145.337200:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000100:00000001:3.0:1786177145.337200:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177145.337205:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000010:0.0:1786177145.337206:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000020:00000040:3.0:1786177145.337206:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177145.337211:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177145.337212:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:0.0:1786177145.337214:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:3.0:1786177145.337214:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177145.337215:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177145.337220:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.337223:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:0.0:1786177145.337225:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177145.337226:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.337229:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177145.337233:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177145.337236:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177145.337241:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177145.337244:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177145.337249:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177145.337251:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.337252:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786177145.337252:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177145.337256:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000040:0.0:1786177145.337260:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:3.0:1786177145.337260:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177145.337261:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177145.337283:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c700.
00000020:00000040:3.0:1786177145.337286:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:0.0:1786177145.337289:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177145.337290:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177145.337292:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.337294:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177145.337295:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177145.337298:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177145.337299:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177145.337302:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:0.0:1786177145.337304:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177145.337305:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177145.337314:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1786177145.337314:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177145.337320:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302044877
00000100:00000001:3.0:1786177145.337324:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177145.337326:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177145.337329:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000040:3.0:1786177145.337330:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e56458ae] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177145.337338:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.337339:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:3.0:1786177145.337340:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177145.337345:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177145.337346:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177145.337347:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302044877
00000100:00080000:3.0:1786177145.337348:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177145.337352:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000010:0.0:1786177145.337355:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134b40.
00000100:00080000:3.0:1786177145.337356:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7368
00000100:00000001:0.0:1786177145.337363:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000400:00000200:3.0:1786177145.337364:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177145.337368:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1786177145.337370:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177145.337376:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177145.337376:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177145.337380:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:3.0:1786177145.337381:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177145.337385:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177145.337390:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00100000:0.0:1786177145.337391:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177145/real 1786177145]  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177145.337392:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177145.337395:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177145.337397:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177145.337399:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177145.337402:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177145.337406:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177145.337410:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177145.337415:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000001:0.0:1786177145.337417:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1786177145.337417:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:0.0:1786177145.337420:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:3.0:1786177145.337421:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:0.0:1786177145.337424:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000100:00000001:3.0:1786177145.337424:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786177145.337429:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000020:00000040:3.0:1786177145.337429:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786177145.337432:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:0.0:1786177145.337433:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:3.0:1786177145.337435:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.337437:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:3.0:1786177145.337438:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177145.337442:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:0.0:1786177145.337443:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177145.337444:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177145.337449:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177145.337452:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177145.337457:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786177145.337458:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177145.337461:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177145.337467:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e300.
00000100:00000001:0.0:1786177145.337468:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000040:3.0:1786177145.337470:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:0.0:1786177145.337471:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177145.337474:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1786177145.337475:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1786177145.337477:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.337478:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177145.337479:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177145.337482:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:0.0:1786177145.337484:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177145.337485:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177145.337490:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:3.0:1786177145.337495:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.337496:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177145.337503:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:0.0:1786177145.337504:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.337532:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177145.337534:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.337536:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177145.337538:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.337540:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177145.337542:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177145.337545:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177145.337548:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177145.337550:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177145.337556:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177145.337561:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177145.337566:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177145.337570:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856342272:192.168.202.110@tcp:250:kworker.0
00000100:00000040:3.0:1786177145.337574:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e3752a60] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.337577:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.337582:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786177145.337587:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177145.337588:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856342272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177145.337592:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7378)
00000100:00000001:0.0:1786177145.337598:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177145.337601:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177145.337604:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177145.337608:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177145.337610:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177145.337612:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177145.337615:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177145.337618:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:0.0:1786177145.337622:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177145.337624:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177145.337626:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:0.0:1786177145.337629:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177145.337631:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.337633:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.337640:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177145.337641:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.337643:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177145.337645:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177145.337647:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177145.337651:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.337665:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177145.337666:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177145.337668:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177145.337669:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177145.337671:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177145.337673:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856342400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177145.337675:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177145.337677:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177145.337678:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177145.337679:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177145.337682:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177145.337683:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177145.337686:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177145.337688:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177145.337691:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177145.337693:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856342400, portal 10
00000100:00000001:0.0:1786177145.337694:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177145.337695:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:0.0:1786177145.337701:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.337705:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177145.337707:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177145.337708:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177145.337710:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856342400, offset 0
00000400:00000200:0.0:1786177145.337713:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177145.337715:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177145.337718:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177145.337720:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177145.337722:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177145.337724:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177145.337725:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177145.337728:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254846:254846:256:4294967295] 192.168.202.110@tcp LPNI seq info [254835:254835:8:4294967295]
00000400:00000200:0.0:1786177145.337731:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177145.337736:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177145.337739:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177145.337741:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177145.337744:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177145.337746:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786177145.337750:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177145.337753:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786177145.337757:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177145.337759:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177145.337762:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177145.337766:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177145.337769:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177145.337771:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177145.337773:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177145.337777:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.337786:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177145.337790:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.337797:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.337799:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177145.337802:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177145.337806:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177145.337808:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177145.337810:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302041098
00000100:00000001:0.0:1786177145.337812:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302044877
00000100:00000001:0.0:1786177145.337813:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302041098
00000100:00000001:0.0:1786177145.337815:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302044877
00000100:00000001:0.0:1786177145.337817:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177145.337819:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.337820:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177145.337823:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177145/real 1786177145]  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177145.337828:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177145.337829:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177145.337831:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177145.337832:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177145.337833:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177145.337834:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177145.337836:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177145.337842:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.337846:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177145.337847:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.337848:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177145.337849:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177145.337851:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.337855:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177145.337858:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.337862:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177145.337864:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.337865:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177145.337866:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.337867:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177145.337868:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177145.337870:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177145.337872:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177145.337874:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177145.337876:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177145.337881:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856342400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177145.337884:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177145.337886:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.337889:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177145.337890:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177145.337892:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177145.337893:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177145.337894:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177145.337896:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856342528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177145.337898:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177145.337899:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177145.337900:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177145.337901:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177145.337903:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177145.337905:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177145.337907:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177145.337908:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177145.337910:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177145.337912:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856342528, portal 10
00000100:00000001:0.0:1786177145.337914:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177145.337915:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:0.0:1786177145.337919:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.337923:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177145.337925:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177145.337926:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177145.337928:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856342528, offset 0
00000400:00000200:0.0:1786177145.337931:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177145.337935:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177145.337939:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177145.337942:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177145.337944:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177145.337947:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177145.337950:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177145.337953:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254847:254847:256:4294967295] 192.168.202.110@tcp LPNI seq info [254836:254836:8:4294967295]
00000400:00000200:0.0:1786177145.337958:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177145.337965:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177145.337970:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177145.337974:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177145.337978:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177145.337981:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786177145.337986:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177145.337988:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786177145.337993:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177145.337995:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177145.337998:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177145.338002:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177145.338006:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177145.338008:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177145.338011:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177145.338015:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338023:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177145.338027:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338033:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.338035:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177145.338038:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177145.338042:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177145.338046:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177145.338048:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302041223
00000100:00000001:0.0:1786177145.338051:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302044877
00000100:00000001:0.0:1786177145.338053:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302041223
00000100:00000001:0.0:1786177145.338055:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302044877
00000100:00000001:0.0:1786177145.338057:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177145.338059:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.338061:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177145.338066:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177145/real 1786177145]  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177145.338072:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177145.338073:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177145.338074:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177145.338076:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177145.338077:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177145.338078:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177145.338081:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177145.338086:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338090:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177145.338091:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338092:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177145.338093:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177145.338095:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338099:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177145.338102:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338107:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177145.338109:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338110:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177145.338110:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338112:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177145.338113:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177145.338114:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177145.338116:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177145.338117:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177145.338120:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177145.338125:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856342528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177145.338127:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177145.338129:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338133:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177145.338134:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177145.338135:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177145.338136:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177145.338138:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177145.338139:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856342656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177145.338141:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177145.338142:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177145.338144:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177145.338145:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177145.338147:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7800.
02000000:00000001:0.0:1786177145.338149:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177145.338151:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177145.338153:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177145.338155:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177145.338157:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856342656, portal 4
00000100:00000001:0.0:1786177145.338159:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177145.338159:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:0.0:1786177145.338164:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338168:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177145.338170:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177145.338171:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177145.338173:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856342656, offset 0
00000400:00000200:0.0:1786177145.338175:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177145.338178:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177145.338180:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177145.338182:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177145.338184:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177145.338185:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177145.338187:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177145.338189:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254848:254848:256:4294967295] 192.168.202.110@tcp LPNI seq info [254837:254837:8:4294967295]
00000400:00000200:0.0:1786177145.338192:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177145.338196:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177145.338199:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177145.338201:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177145.338204:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177145.338206:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786177145.338208:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177145.338210:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786177145.338212:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177145.338214:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177145.338216:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177145.338218:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177145.338220:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177145.338222:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177145.338223:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177145.338226:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338231:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177145.338233:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338237:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.338238:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177145.338240:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177145.338242:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177145.338244:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177145.338245:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302041098
00000100:00000001:0.0:1786177145.338247:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302044877
00000100:00000001:0.0:1786177145.338248:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302041098
00000100:00000001:0.0:1786177145.338250:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302044877
00000100:00000001:0.0:1786177145.338251:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177145.338252:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.338253:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177145.338255:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177145/real 1786177145]  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177145.338260:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177145.338260:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177145.338274:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177145.338277:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177145.338278:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177145.338280:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177145.338282:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177145.338288:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338292:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177145.338293:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338295:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177145.338295:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177145.338298:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338302:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177145.338307:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338313:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177145.338315:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338317:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177145.338318:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338319:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177145.338321:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177145.338324:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177145.338327:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177145.338330:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177145.338334:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177145.338344:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856342656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177145.338348:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177145.338352:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338359:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177145.338360:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177145.338362:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177145.338363:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177145.338365:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177145.338367:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856342784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177145.338369:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177145.338370:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177145.338372:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177145.338373:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177145.338375:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177145.338377:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177145.338379:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177145.338381:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177145.338383:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177145.338385:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856342784, portal 4
00000100:00000001:0.0:1786177145.338387:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177145.338387:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:0.0:1786177145.338392:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338396:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177145.338398:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177145.338399:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177145.338401:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856342784, offset 0
00000400:00000200:0.0:1786177145.338403:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177145.338406:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177145.338408:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177145.338410:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177145.338411:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177145.338413:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177145.338415:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177145.338417:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254849:254849:256:4294967295] 192.168.202.110@tcp LPNI seq info [254838:254838:8:4294967295]
00000400:00000200:0.0:1786177145.338419:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177145.338424:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177145.338427:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177145.338429:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177145.338432:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177145.338434:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786177145.338436:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177145.338438:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786177145.338441:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177145.338442:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177145.338444:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177145.338446:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177145.338448:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177145.338449:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177145.338451:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177145.338453:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338458:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177145.338460:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338464:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.338465:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177145.338467:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177145.338469:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177145.338471:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177145.338473:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302041106
00000100:00000001:0.0:1786177145.338474:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302044877
00000100:00000001:0.0:1786177145.338476:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302041106
00000100:00000001:0.0:1786177145.338477:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302044877
00000100:00000001:0.0:1786177145.338479:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177145.338480:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.338481:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177145.338483:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177145/real 1786177145]  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177145.338489:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177145.338489:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177145.338491:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177145.338492:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177145.338493:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177145.338494:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177145.338496:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177145.338501:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338505:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177145.338506:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338507:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177145.338508:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177145.338527:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338531:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177145.338534:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338538:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177145.338540:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338541:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177145.338542:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338543:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177145.338544:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177145.338545:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177145.338547:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177145.338548:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177145.338551:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177145.338556:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856342784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177145.338559:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338560:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177145.338562:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856342400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338566:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177145.338567:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177145.338568:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177145.338570:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177145.338571:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177145.338572:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177145.338573:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177145.338574:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786177145.338576:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177145.338577:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177145.338578:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:0.0:1786177145.338580:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177145.338581:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338582:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177145.338584:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856342528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338588:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177145.338589:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177145.338590:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177145.338591:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177145.338592:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177145.338592:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177145.338593:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177145.338594:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786177145.338596:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177145.338597:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177145.338598:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:0.0:1786177145.338599:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177145.338600:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338601:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177145.338603:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856342656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338607:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177145.338607:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177145.338608:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7800.
02000000:00000001:0.0:1786177145.338610:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177145.338610:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177145.338611:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177145.338612:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177145.338613:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177145.338614:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177145.338615:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177145.338615:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:0.0:1786177145.338616:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177145.338617:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338618:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177145.338620:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856342784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177145.338624:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177145.338624:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177145.338625:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177145.338626:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177145.338627:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177145.338628:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177145.338629:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177145.338629:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786177145.338631:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177145.338632:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177145.338633:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:0.0:1786177145.338634:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177145.338634:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338636:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177145.338639:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177145.338639:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177145.338641:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177145.338642:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177145.912607:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177145.912612:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177145.912616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177145.912617:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177145.912619:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177145.912620:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177145.912681:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177145.912685:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177145.912688:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177146.360608:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177146.360614:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177146.360615:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177146.360619:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177146.360622:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177146.360625:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177146.360627:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177146.360629:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177146.360631:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177146.936597:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177146.936603:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177146.936607:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177146.936607:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177146.936610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177146.936611:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177146.936665:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177146.936668:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177146.936671:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177147.384600:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177147.384607:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177147.384609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177147.384614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177147.384618:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177147.384621:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177147.384625:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177147.384628:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177147.384632:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177147.960610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177147.960615:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177147.960621:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177147.960623:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177147.960626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177147.960628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177147.960692:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177147.960697:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177147.960701:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177148.408616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177148.408629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177148.408632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177148.408639:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177148.408643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177148.408653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177148.408656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177148.408659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177148.408663:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177148.984590:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177148.984595:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177148.984599:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177148.984600:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177148.984603:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177148.984604:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177148.984660:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177148.984663:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177148.984666:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177149.176603:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302044877, at 4302044928
00000100:00000040:1.0:1786177149.176603:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302044877, at 4302044928
00000100:00000010:0.0:1786177149.176618:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000010:1.0:1786177149.176626:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134b40.
00000100:00000001:2.0:1786177149.432595:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177149.432602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177149.432604:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177149.432610:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177149.432615:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177149.432618:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177149.432621:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177149.432624:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177149.432627:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177150.008590:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177150.008595:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177150.008598:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177150.008600:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177150.008602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177150.008604:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177150.008666:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177150.008670:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177150.008673:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177150.456603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177150.456609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177150.456610:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177150.456615:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177150.456618:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177150.456621:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177150.456623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177150.456625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177150.456628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177150.456635:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177150.456643:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177150.456645:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177150.456649:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177150.456653:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177150.456659:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7373
00000400:00000200:0.0:1786177150.456666:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177150.456671:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177150.456673:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177150.456675:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.456678:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177150.456679:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.456681:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177150.456683:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177150.456684:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177150.456686:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177150.456688:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177150.456689:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177150.456691:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177150.456693:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.456695:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177150.456696:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.456697:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177150.456699:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177150.456701:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177150.456703:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177150.456705:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177150.456707:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177150.456708:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.456710:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177150.456711:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177150.456718:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41bb80.
00000020:00000040:0.0:1786177150.456721:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177150.456723:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177150.456725:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.456727:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177150.456729:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177150.456732:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177150.456737:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4000.
00000100:00000001:0.0:1786177150.456741:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177150.456747:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177150.456755:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177150.456758:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177150.456761:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9983a05] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177150.456778:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177150.456781:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177150.456784:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177150.456785:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177150.456787:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177150.456790:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177150.456794:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7373
00000400:00000200:0.0:1786177150.456798:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177150.456802:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177150.456804:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1786177150.456805:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177150.456809:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1786177150.456810:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177150.456813:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1786177150.456814:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177150.456819:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.456822:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:3.0:1786177150.456823:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177150.456824:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177150.456827:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177150.456829:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177150.456832:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177150.456835:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177150.456838:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177150.456844:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177150.456845:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177150.456846:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177150.456848:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177150.456849:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.456853:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177150.456854:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
02000000:00000001:3.0:1786177150.456854:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1786177150.456859:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1786177150.456859:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177150.456865:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177150.456865:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177150.456866:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177150.456873:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1786177150.456873:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856342912:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1786177150.456877:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.456880:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177150.456882:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:0.0:1786177150.456884:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177150.456886:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.456888:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177150.456888:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177150.456893:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177150.456894:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419c00.
00000020:00000040:0.0:1786177150.456898:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177150.456898:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1786177150.456902:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177150.456903:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.456904:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177150.456905:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177150.456906:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177150.456909:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5800.
00000100:00000001:0.0:1786177150.456911:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177150.456914:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00100000:0.0:1786177150.456915:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1786177150.456918:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.456926:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177150.456929:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177150.456930:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177150.456937:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cf890c1c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1786177150.456937:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177150.456945:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177150.456946:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177150.456951:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177150.456953:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177150.456954:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177150.456955:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177150.456956:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177150.456959:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7373
00000100:00000200:3.0:1786177150.456959:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856342912, portal 25
00000100:00000001:3.0:1786177150.456964:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177150.456967:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761670016 : -114547947881600 : ffff97d1ba41bb80)
00000400:00000200:0.0:1786177150.456968:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177150.456974:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1786177150.456977:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177150.456978:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177150.456990:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177150.456993:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177150.456997:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000040:0.0:1786177150.456999:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:3.0:1786177150.457001:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856342912, offset 0
00000100:00000001:0.0:1786177150.457004:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177150.457007:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.457010:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177150.457011:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:0.0:1786177150.457013:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177150.457016:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177150.457017:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177150.457018:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177150.457020:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177150.457024:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177150.457024:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177150.457029:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.457031:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177150.457034:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:0.0:1786177150.457036:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.457040:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177150.457042:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:0.0:1786177150.457044:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:3.0:1786177150.457048:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:0.0:1786177150.457052:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1786177150.457055:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1786177150.457058:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177150.457062:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1786177150.457063:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:0.0:1786177150.457068:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177150.457069:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254850:254850:256:4294967295] 192.168.202.110@tcp LPNI seq info [254839:254839:8:4294967295]
02000000:00000001:0.0:1786177150.457073:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.457076:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:3.0:1786177150.457079:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:0.0:1786177150.457080:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177150.457083:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.457084:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177150.457088:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a300.
00000020:00000040:0.0:1786177150.457092:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177150.457094:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177150.457095:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.457096:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177150.457097:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177150.457098:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177150.457103:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000010:0.0:1786177150.457106:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7800.
00000100:00000001:0.0:1786177150.457111:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177150.457112:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177150.457119:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00100000:0.0:1786177150.457120:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.0:1786177150.457131:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177150.457136:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000200:3.0:1786177150.457138:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:0.0:1786177150.457141:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177150.457146:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786177150.457149:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b0d3569] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1786177150.457151:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:0.0:1786177150.457162:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1786177150.457162:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177150.457168:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:3.0:1786177150.457168:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00080000:0.0:1786177150.457173:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177150.457174:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177150.457176:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1786177150.457177:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1786177150.457179:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177150.457183:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7373
00000400:00000200:3.0:1786177150.457188:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177150.457191:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177150.457195:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177150.457197:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1786177150.457197:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000100:00000040:0.0:1786177150.457202:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:3.0:1786177150.457202:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:0.0:1786177150.457207:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177150.457209:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177150.457209:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1786177150.457211:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:3.0:1786177150.457227:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177150.457229:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177150.457231:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177150.457233:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177150.457237:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177150.457241:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177150.457243:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:3.0:1786177150.457246:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177150.457247:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.457249:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177150.457253:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.457254:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177150.457255:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177150.457257:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177150.457258:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177150.457259:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177150.457260:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177150.457264:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177150.457269:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.457272:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177150.457275:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177150.457277:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1786177150.457278:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.457281:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.457281:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000010:0.0:1786177150.457286:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a680.
00000020:00000040:0.0:1786177150.457288:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177150.457289:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177150.457290:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177150.457292:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.457295:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177150.457296:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177150.457299:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177150.457299:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177150.457305:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302049997
02000000:00000010:0.0:1786177150.457306:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000100:00000001:0.0:1786177150.457312:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177150.457315:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37de0.
00000100:00100000:0.0:1786177150.457316:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177150.457323:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177150.457323:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786177150.457325:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177150.457330:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000911ee1e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.457330:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302049997
00000100:00000010:3.0:1786177150.457336:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37c00.
00000100:00000001:0.0:1786177150.457338:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177150.457341:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177150.457341:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00080000:0.0:1786177150.457347:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177150.457347:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177150.457349:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177150.457352:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177150.457353:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.457356:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177150.457357:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00080000:0.0:1786177150.457360:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7373
00000400:00000200:0.0:1786177150.457365:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1786177150.457365:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177150/real 1786177150]  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177150.457370:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177150.457374:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177150.457380:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.457385:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:3.0:1786177150.457388:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:0.0:1786177150.457389:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000001:3.0:1786177150.457392:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1786177150.457394:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000010:3.0:1786177150.457395:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000020:00000001:0.0:1786177150.457400:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:3.0:1786177150.457401:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000020:00000040:0.0:1786177150.457404:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000400:00000010:3.0:1786177150.457405:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000020:00000001:0.0:1786177150.457408:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177150.457409:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:0.0:1786177150.457412:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177150.457417:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000200:3.0:1786177150.457417:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177150.457423:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786177150.457431:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177150.457435:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177150.457439:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:3.0:1786177150.457440:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177150.457441:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177150.457446:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177150.457450:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177150.457455:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177150.457458:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177150.457462:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177150.457463:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:0.0:1786177150.457467:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.457467:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177150.457472:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177150.457473:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000008:00100000:0.0:1786177150.457474:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177150.457476:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:0.0:1786177150.457478:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177150.457480:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177150.457485:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1786177150.457486:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419500.
00000020:00000040:0.0:1786177150.457491:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177150.457497:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177150.457499:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.457501:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177150.457503:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177150.457506:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:3.0:1786177150.457508:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.457535:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177150.457539:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.457540:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177150.457542:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.457545:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177150.457547:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177150.457549:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:0.0:1786177150.457552:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177150.457553:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177150.457558:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000010:0.0:1786177150.457559:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000040:3.0:1786177150.457563:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177150.457565:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177150.457577:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1786177150.457578:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856342912:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177150.457588:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.457592:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177150.457598:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41bb80 x1872934856342912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177150.457603:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177150.457608:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177150.457617:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b6718a9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.457621:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177150.457625:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177150.457629:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
00000100:00000001:0.0:1786177150.457633:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177150.457634:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177150.457638:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177150.457639:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000040:0.0:1786177150.457641:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7383)
00000020:00000001:3.0:1786177150.457642:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177150.457646:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4000.
02000000:00000001:3.0:1786177150.457650:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177150.457651:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177150.457653:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41bb80.
00000100:00000001:3.0:1786177150.457656:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177150.457658:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.457668:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.457674:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177150.457676:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.457679:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177150.457680:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177150.457682:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177150.457687:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.457694:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177150.457696:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177150.457698:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177150.457699:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177150.457701:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177150.457704:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177150.457708:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177150.457710:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177150.457712:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177150.457714:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177150.457717:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177150.457720:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177150.457724:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177150.457728:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177150.457732:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177150.457735:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856343040, portal 10
00000100:00000001:3.0:1786177150.457736:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177150.457738:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761661952 : -114547947889664 : ffff97d1ba419c00)
00000100:00000040:3.0:1786177150.457745:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.457751:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177150.457754:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177150.457756:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177150.457758:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856343040, offset 0
00000400:00000200:3.0:1786177150.457761:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177150.457765:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177150.457768:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177150.457770:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177150.457773:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177150.457775:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177150.457778:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177150.457781:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254851:254851:256:4294967295] 192.168.202.110@tcp LPNI seq info [254840:254840:8:4294967295]
00000400:00000200:3.0:1786177150.457785:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177150.457792:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177150.457796:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177150.457799:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177150.457802:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177150.457805:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177150.457810:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177150.457812:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177150.457816:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177150.457818:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177150.457821:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177150.457824:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177150.457827:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177150.457829:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177150.457831:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177150.457835:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.457842:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177150.457845:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.457852:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.457854:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177150.457856:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177150.457860:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177150.457863:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177150.457864:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302046218
00000100:00000001:3.0:1786177150.457867:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302049997
00000100:00000001:3.0:1786177150.457869:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302046218
00000100:00000001:3.0:1786177150.457871:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302049997
00000100:00000001:3.0:1786177150.457873:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177150.457875:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.457877:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177150.457880:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177150/real 1786177150]  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177150.457888:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177150.457889:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177150.457891:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177150.457893:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177150.457895:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177150.457897:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177150.457900:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177150.457909:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.457920:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177150.457922:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.457923:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177150.457925:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177150.457928:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.457934:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177150.457937:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.457944:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177150.457946:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.457948:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177150.457949:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.457951:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177150.457953:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177150.457955:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177150.457958:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177150.457960:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177150.457964:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177150.457972:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177150.457975:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177150.457978:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.457985:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177150.457986:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177150.457988:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177150.457990:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177150.457992:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177150.457995:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177150.457997:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177150.457999:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177150.458000:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177150.458002:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177150.458005:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177150.458007:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177150.458010:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177150.458012:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177150.458015:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177150.458017:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856343168, portal 10
00000100:00000001:3.0:1786177150.458019:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177150.458020:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761663744 : -114547947887872 : ffff97d1ba41a300)
00000100:00000040:3.0:1786177150.458026:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458032:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177150.458034:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177150.458036:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177150.458038:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856343168, offset 0
00000400:00000200:3.0:1786177150.458041:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177150.458044:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177150.458047:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177150.458050:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177150.458052:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177150.458054:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177150.458056:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177150.458059:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254852:254852:256:4294967295] 192.168.202.110@tcp LPNI seq info [254841:254841:8:4294967295]
00000400:00000200:3.0:1786177150.458063:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177150.458069:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177150.458073:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177150.458076:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177150.458080:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177150.458082:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177150.458087:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177150.458089:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177150.458093:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177150.458095:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177150.458097:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177150.458101:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177150.458104:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177150.458105:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177150.458108:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177150.458111:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458118:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177150.458121:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458127:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.458129:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177150.458132:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177150.458134:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177150.458137:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177150.458139:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302046343
00000100:00000001:3.0:1786177150.458141:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302049997
00000100:00000001:3.0:1786177150.458143:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302046343
00000100:00000001:3.0:1786177150.458145:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302049997
00000100:00000001:3.0:1786177150.458147:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177150.458149:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.458150:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177150.458154:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177150/real 1786177150]  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177150.458162:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177150.458163:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177150.458165:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177150.458167:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177150.458169:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177150.458170:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177150.458173:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177150.458181:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458187:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177150.458189:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458190:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177150.458191:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177150.458195:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458201:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177150.458204:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458211:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177150.458225:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458227:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177150.458228:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458230:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177150.458232:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177150.458234:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177150.458238:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177150.458240:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177150.458244:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177150.458251:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177150.458255:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177150.458258:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458264:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177150.458266:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177150.458268:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177150.458269:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177150.458271:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177150.458274:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177150.458277:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177150.458278:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177150.458280:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177150.458281:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177150.458285:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177150.458286:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177150.458289:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177150.458292:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177150.458295:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177150.458297:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856343296, portal 4
00000100:00000001:3.0:1786177150.458299:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177150.458300:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761664640 : -114547947886976 : ffff97d1ba41a680)
00000100:00000040:3.0:1786177150.458306:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458312:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177150.458314:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177150.458316:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177150.458318:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856343296, offset 0
00000400:00000200:3.0:1786177150.458322:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177150.458325:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177150.458328:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177150.458331:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177150.458334:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177150.458336:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177150.458338:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177150.458341:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254853:254853:256:4294967295] 192.168.202.110@tcp LPNI seq info [254842:254842:8:4294967295]
00000400:00000200:3.0:1786177150.458345:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177150.458351:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177150.458355:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177150.458358:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177150.458362:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177150.458365:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177150.458368:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177150.458371:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177150.458374:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177150.458376:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177150.458379:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177150.458382:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177150.458384:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177150.458386:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177150.458388:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177150.458392:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458399:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177150.458402:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458408:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.458410:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177150.458412:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177150.458415:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177150.458418:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177150.458420:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302046218
00000100:00000001:3.0:1786177150.458422:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302049997
00000100:00000001:3.0:1786177150.458424:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302046218
00000100:00000001:3.0:1786177150.458426:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302049997
00000100:00000001:3.0:1786177150.458428:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177150.458430:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.458431:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177150.458435:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177150/real 1786177150]  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177150.458443:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177150.458444:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177150.458446:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177150.458448:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177150.458449:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177150.458451:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177150.458454:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177150.458463:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458469:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177150.458470:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458472:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177150.458473:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177150.458476:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458483:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177150.458486:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458493:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177150.458496:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458497:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177150.458498:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458500:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177150.458501:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177150.458504:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177150.458506:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177150.458508:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177150.458527:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177150.458534:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177150.458537:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177150.458540:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458547:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177150.458549:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177150.458551:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177150.458552:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177150.458554:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177150.458556:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177150.458559:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177150.458561:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177150.458562:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177150.458564:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177150.458567:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177150.458569:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177150.458572:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177150.458574:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177150.458577:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177150.458579:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856343424, portal 4
00000100:00000001:3.0:1786177150.458581:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177150.458583:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761660160 : -114547947891456 : ffff97d1ba419500)
00000100:00000040:3.0:1786177150.458588:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458594:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177150.458596:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177150.458598:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177150.458600:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856343424, offset 0
00000400:00000200:3.0:1786177150.458604:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177150.458607:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177150.458610:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177150.458612:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177150.458614:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177150.458617:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177150.458619:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177150.458622:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254854:254854:256:4294967295] 192.168.202.110@tcp LPNI seq info [254843:254843:8:4294967295]
00000400:00000200:3.0:1786177150.458626:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177150.458631:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177150.458636:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177150.458639:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177150.458642:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177150.458645:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177150.458649:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177150.458651:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177150.458655:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177150.458657:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177150.458660:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177150.458663:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177150.458665:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177150.458667:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177150.458669:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177150.458672:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458686:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177150.458689:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458695:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.458697:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177150.458699:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177150.458702:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177150.458705:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177150.458707:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302046227
00000100:00000001:3.0:1786177150.458709:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302049997
00000100:00000001:3.0:1786177150.458711:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302046227
00000100:00000001:3.0:1786177150.458713:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302049997
00000100:00000001:3.0:1786177150.458715:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177150.458717:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.458719:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177150.458722:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177150/real 1786177150]  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177150.458729:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177150.458730:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177150.458732:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177150.458734:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177150.458735:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177150.458737:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177150.458740:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177150.458748:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458755:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177150.458756:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458758:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177150.458760:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177150.458763:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458770:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177150.458773:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458779:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177150.458781:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458783:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177150.458784:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458786:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177150.458788:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177150.458790:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177150.458792:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177150.458794:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177150.458798:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177150.458804:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177150.458807:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458809:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177150.458812:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419c00 x1872934856343040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458819:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177150.458820:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177150.458822:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177150.458824:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177150.458825:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177150.458827:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177150.458829:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177150.458830:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5800.
02000000:00000001:3.0:1786177150.458833:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177150.458834:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177150.458836:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419c00.
00000100:00000001:3.0:1786177150.458838:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177150.458840:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458841:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177150.458844:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a300 x1872934856343168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458851:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177150.458852:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177150.458853:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177150.458855:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177150.458856:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177150.458858:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177150.458860:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177150.458861:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7800.
02000000:00000001:3.0:1786177150.458863:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177150.458864:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177150.458865:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a300.
00000100:00000001:3.0:1786177150.458867:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177150.458868:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458869:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177150.458872:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a680 x1872934856343296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458878:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177150.458879:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177150.458881:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177150.458883:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177150.458884:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177150.458885:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177150.458887:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177150.458888:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:3.0:1786177150.458889:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177150.458891:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177150.458892:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a680.
00000100:00000001:3.0:1786177150.458894:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177150.458895:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458896:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177150.458900:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419500 x1872934856343424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177150.458905:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177150.458906:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177150.458907:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177150.458909:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177150.458910:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177150.458911:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177150.458913:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177150.458914:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:3.0:1786177150.458916:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177150.458917:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177150.458918:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419500.
00000100:00000001:3.0:1786177150.458920:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177150.458921:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458923:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177150.458925:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177150.458927:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177150.458928:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177150.458930:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177151.032571:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177151.032576:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177151.032578:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177151.032580:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177151.032583:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177151.032585:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177151.032586:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177151.032588:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177151.032590:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177151.032591:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177151.032592:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177151.032593:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177151.032594:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177151.032595:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177151.032596:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177151.032597:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177151.032599:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177151.032600:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177151.032646:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177151.032649:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177151.032651:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177151.480641:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177151.480647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177151.480649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177151.480653:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177151.480655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177151.480659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177151.480661:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177151.480663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177151.480665:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177152.056632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177152.056639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177152.056644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177152.056645:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177152.056649:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177152.056650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177152.056698:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177152.056702:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177152.056705:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177152.504601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177152.504609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177152.504611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177152.504616:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177152.504619:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177152.504622:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177152.504625:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177152.504627:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177152.504630:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177153.080602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177153.080608:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177153.080620:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177153.080621:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177153.080624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177153.080627:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177153.080629:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177153.080632:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177153.080635:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177153.080638:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177153.080639:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177153.080642:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177153.080700:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177153.080704:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177153.080707:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177153.528612:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177153.528621:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177153.528623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177153.528629:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177153.528632:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177153.528636:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177153.528639:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177153.528642:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177153.528645:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177154.104602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177154.104609:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177154.104614:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177154.104616:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177154.104619:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177154.104620:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177154.104672:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177154.104675:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177154.104677:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177154.296616:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302049997, at 4302050048
00000100:00000040:1.0:1786177154.296616:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302049997, at 4302050048
00000100:00000010:1.0:1786177154.296629:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37c00.
00000100:00000010:0.0:1786177154.296631:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37de0.
00000100:00000001:2.0:1786177154.552609:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177154.552617:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177154.552619:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177154.552625:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177154.552628:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177154.552632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177154.552635:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177154.552637:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177154.552640:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177155.128581:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177155.128588:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177155.128600:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177155.128601:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177155.128604:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177155.128605:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177155.128663:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177155.128667:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177155.128670:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177155.576638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177155.576644:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177155.576646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177155.576651:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177155.576654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1786177155.576658:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786177155.576658:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177155.576663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177155.576666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177155.576668:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177155.576670:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177155.576672:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177155.576677:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177155.576680:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177155.576706:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7378
00000400:00000200:0.0:1786177155.576715:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177155.576721:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177155.576724:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177155.576727:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177155.576730:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177155.576733:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177155.576735:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177155.576739:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177155.576741:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177155.576744:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177155.576748:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177155.576749:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177155.576751:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177155.576753:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177155.576754:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177155.576755:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177155.576756:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177155.576758:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177155.576765:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177155.576767:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177155.576770:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177155.576773:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177155.576776:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177155.576778:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177155.576780:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177155.576790:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41aa00.
00000020:00000040:0.0:1786177155.576810:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177155.576815:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177155.576816:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177155.576819:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177155.576821:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177155.576825:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177155.576832:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000100:00000001:0.0:1786177155.576838:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177155.576847:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177155.576856:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177155.576860:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177155.576872:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da0255a4] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177155.576914:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177155.576918:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177155.576923:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177155.576925:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177155.576927:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177155.576930:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177155.576933:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7378
00000400:00000200:0.0:1786177155.576939:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177155.576943:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177155.576945:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177155.576949:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177155.576951:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177155.576953:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177155.576956:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177155.576956:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1786177155.576964:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177155.576964:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177155.576970:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1786177155.576971:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177155.576976:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177155.576983:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000040:3.0:1786177155.576984:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177155.576989:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177155.577003:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:3.0:1786177155.577011:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177155.577013:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177155.577016:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177155.577022:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1786177155.577026:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177155.577028:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177155.577038:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856343552:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177155.577049:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1786177155.577055:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1786177155.577055:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177155.577058:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177155.577063:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:0.0:1786177155.577064:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177155.577069:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000010:3.0:1786177155.577072:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
00000020:00000040:0.0:1786177155.577073:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
02000000:00000001:3.0:1786177155.577077:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177155.577084:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1786177155.577087:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177155.577094:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000001:0.0:1786177155.577101:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177155.577103:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
02000000:00000001:0.0:1786177155.577108:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1786177155.577113:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856343552, portal 25
02000000:00000001:0.0:1786177155.577115:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177155.577118:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177155.577120:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761665536 : -114547947886080 : ffff97d1ba41aa00)
02000000:00000001:0.0:1786177155.577125:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177155.577127:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177155.577130:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177155.577140:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000008:00100000:0.0:1786177155.577141:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000040:3.0:1786177155.577149:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177155.577154:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177155.577158:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856343552, offset 0
00000008:00000001:0.0:1786177155.577183:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177155.577187:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.577191:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000010:0.0:1786177155.577192:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41ad80.
00000020:00000040:0.0:1786177155.577198:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177155.577202:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1786177155.577204:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786177155.577206:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177155.577211:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177155.577213:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1786177155.577214:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177155.577220:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177155.577225:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:0.0:1786177155.577226:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.577230:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177155.577233:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000010:0.0:1786177155.577235:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000400:00000200:3.0:1786177155.577241:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254855:254855:256:4294967295] 192.168.202.110@tcp LPNI seq info [254844:254844:8:4294967295]
00000100:00000001:0.0:1786177155.577245:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.577248:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177155.577259:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177155.577273:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00100000:0.0:1786177155.577274:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:3.0:1786177155.577279:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177155.577293:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177155.577301:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786177155.577312:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177155.577317:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:0.0:1786177155.577319:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177155.577325:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1786177155.577326:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177155.577332:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00000040:0.0:1786177155.577338:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034e2c676] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177155.577342:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177155.577358:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1786177155.577375:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.577375:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177155.577378:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177155.577384:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177155.577388:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177155.577391:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177155.577408:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177155.577414:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177155.577416:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177155.577417:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177155.577427:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:3.0:1786177155.577428:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.577435:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177155.577436:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7378
00000100:00000001:3.0:1786177155.577444:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000200:0.0:1786177155.577448:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177155.577452:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177155.577454:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177155.577457:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.577464:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177155.577468:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177155.577470:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302055117
00000100:00000001:0.0:1786177155.577478:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:3.0:1786177155.577481:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37600.
00000100:00000040:0.0:1786177155.577482:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177155.577492:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177155.577494:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177155.577495:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177155.577499:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177155.577500:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000040:0.0:1786177155.577545:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000001:3.0:1786177155.577547:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302055118
00000100:00000010:3.0:1786177155.577554:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad378a0.
00000020:00000001:0.0:1786177155.577557:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:3.0:1786177155.577561:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000020:00000001:0.0:1786177155.577562:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:3.0:1786177155.577572:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1786177155.577574:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177155.577580:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177155.577581:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177155.577585:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177155.577590:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1786177155.577591:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177155.577596:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00100000:3.0:1786177155.577602:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177155/real 1786177155]  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177155.577603:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177155.577619:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177155.577620:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:0.0:1786177155.577623:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000001:3.0:1786177155.577625:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:0.0:1786177155.577629:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:3.0:1786177155.577631:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
02000000:00000001:0.0:1786177155.577634:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.577638:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177155.577642:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177155.577643:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177155.577645:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000008:00100000:0.0:1786177155.577647:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177155.577653:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177155.577653:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177155.577666:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177155.577673:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b480.
00000100:00000200:3.0:1786177155.577676:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177155.577677:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177155.577681:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177155.577684:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177155.577688:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177155.577694:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177155.577701:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.577701:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177155.577708:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:0.0:1786177155.577709:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
00000100:00000001:0.0:1786177155.577720:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.577728:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00100000:0.0:1786177155.577730:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177155.577731:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177155.577748:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.577779:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177155.577786:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1786177155.577789:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177155.577790:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177155.577802:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001215e9e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.577823:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177155.577831:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.577834:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177155.577835:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.577836:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.577840:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177155.577842:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177155.577845:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177155.577847:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177155.577852:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177155.577856:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00080000:0.0:1786177155.577860:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000040:3.0:1786177155.577865:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177155.577873:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1786177155.577874:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856343552:192.168.202.110@tcp:250:kworker.0
00000100:00080000:0.0:1786177155.577879:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177155.577885:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177155.577886:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177155.577888:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177155.577892:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41aa00 x1872934856343552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.577897:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00080000:0.0:1786177155.577902:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7378
02000000:00000001:3.0:1786177155.577903:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177155.577906:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177155.577913:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000400:00000200:0.0:1786177155.577918:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1786177155.577919:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177155.577922:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177155.577925:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177155.577928:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:3.0:1786177155.577931:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177155.577933:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177155.577935:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41aa00.
00000400:00000200:0.0:1786177155.577937:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177155.577940:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177155.577943:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177155.577946:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177155.577946:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177155.577954:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177155.577957:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177155.577962:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177155.577962:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.577965:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177155.577969:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1786177155.577970:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177155.577972:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1786177155.577978:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:3.0:1786177155.577978:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.577990:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177155.577991:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177155.577993:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177155.577994:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1786177155.577996:0:62275:0:(genops.c:813:class_conn2export()) Process entered
02000000:00000001:3.0:1786177155.577997:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177155.578000:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00100000:3.0:1786177155.578002:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343680:192.168.202.110@tcp:38:kworker.0
00000020:00000001:0.0:1786177155.578007:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177155.578009:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177155.578012:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177155.578013:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177155.578015:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177155.578018:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177155.578020:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.578023:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177155.578025:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000020:00000040:0.0:1786177155.578028:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000400:00000010:3.0:1786177155.578029:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000020:00000001:0.0:1786177155.578035:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000200:3.0:1786177155.578035:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856343680, portal 10
00000100:00000001:3.0:1786177155.578042:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177155.578047:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761666432 : -114547947885184 : ffff97d1ba41ad80)
00000020:00000001:0.0:1786177155.578049:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177155.578056:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:3.0:1786177155.578056:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177155.578067:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177155.578072:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177155.578082:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:3.0:1786177155.578083:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177155.578091:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000001:0.0:1786177155.578092:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177155.578098:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000200:3.0:1786177155.578107:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856343680, offset 0
00000100:00080000:0.0:1786177155.578113:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1786177155.578114:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177155.578123:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177155.578124:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.578129:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:0.0:1786177155.578130:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1786177155.578134:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177155.578136:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177155.578138:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177155.578141:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:0.0:1786177155.578142:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177155.578152:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.578152:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254856:254856:256:4294967295] 192.168.202.110@tcp LPNI seq info [254845:254845:8:4294967295]
00000400:00000200:3.0:1786177155.578160:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177155.578162:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177155.578176:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177155.578186:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.578187:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177155.578194:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177155.578199:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177155.578203:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000010:0.0:1786177155.578205:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b100.
00000020:00000040:0.0:1786177155.578212:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000800:00000200:3.0:1786177155.578212:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177155.578220:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
02000000:00000001:0.0:1786177155.578221:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177155.578225:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177155.578231:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:3.1:1786177155.578231:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:0.0:1786177155.578235:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:3.1:1786177155.578237:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
02000000:00000001:0.0:1786177155.578239:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1786177155.578256:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000010:0.0:1786177155.578257:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000800:00000010:3.0:1786177155.578263:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00000001:0.0:1786177155.578264:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.578270:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:0.0:1786177155.578276:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177155.578281:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177155.578291:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177155.578299:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:0.0:1786177155.578312:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177155.578312:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1786177155.578322:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177155.578326:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177155.578333:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000fa632fb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.578363:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177155.578369:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177155.578370:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177155.578380:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177155.578392:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177155.578393:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177155.578399:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177155.578400:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177155.578407:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177155.578409:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177155.578411:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:3.0:1786177155.578414:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000100:00080000:0.0:1786177155.578416:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7378
00000400:00000200:3.0:1786177155.578436:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177155.578437:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177155.578449:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302051338
00000400:00000200:0.0:1786177155.578454:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177155.578458:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302055117
00000100:00080000:0.0:1786177155.578459:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177155.578462:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302051338
00000100:00000040:0.0:1786177155.578466:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177155.578468:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302055118
00000100:00000001:3.0:1786177155.578476:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177155.578477:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177155.578485:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177155.578488:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177155.578491:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1786177155.578493:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177155.578502:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00100000:3.0:1786177155.578502:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177155/real 1786177155]  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177155.578539:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177155.578544:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177155.578549:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177155.578552:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000001:3.0:1786177155.578552:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177155.578555:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000001:0.0:1786177155.578557:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000010:3.0:1786177155.578559:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00000040:0.0:1786177155.578564:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177155.578565:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000100:00000001:0.0:1786177155.578575:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1786177155.578576:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000040:0.0:1786177155.578583:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177155.578583:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1786177155.578589:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:3.0:1786177155.578594:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177155.578595:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177155.578603:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177155.578613:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177155.578619:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1786177155.578623:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177155.578624:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177155.578633:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177155.578639:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177155.578642:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177155.578653:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177155.578655:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:0.0:1786177155.578657:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.578660:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177155.578661:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.578669:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:0.0:1786177155.578670:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419180.
00000020:00000040:0.0:1786177155.578677:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177155.578680:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:3.0:1786177155.578682:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177155.578686:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177155.578691:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177155.578694:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177155.578696:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177155.578705:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000100:00000001:3.0:1786177155.578712:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177155.578715:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177155.578721:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177155.578722:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177155.578743:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177155.578747:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.578753:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177155.578758:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.578762:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177155.578765:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177155.578772:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177155.578778:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177155.578783:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177155.578785:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1786177155.578790:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177155.578797:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177155.578810:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005a89d8ab] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177155.578816:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343680:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177155.578830:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1786177155.578837:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177155.578840:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177155.578845:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7389)
00000100:00000001:3.0:1786177155.578865:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177155.578867:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177155.578870:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177155.578873:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177155.578876:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177155.578880:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177155.578883:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177155.578886:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177155.578887:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177155.578889:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177155.578893:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177155.578895:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.578898:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177155.578901:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177155.578904:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177155.578907:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856343808, portal 10
00000100:00000001:3.0:1786177155.578908:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177155.578909:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761668224 : -114547947883392 : ffff97d1ba41b480)
00000100:00000040:3.0:1786177155.578917:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.578922:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177155.578924:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177155.578925:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177155.578927:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856343808, offset 0
00000400:00000200:3.0:1786177155.578930:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177155.578933:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177155.578936:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177155.578938:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177155.578940:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177155.578942:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177155.578944:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177155.578947:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254857:254857:256:4294967295] 192.168.202.110@tcp LPNI seq info [254846:254846:8:4294967295]
00000400:00000200:3.0:1786177155.578950:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177155.578955:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177155.578958:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177155.578961:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177155.578964:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177155.578966:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786177155.578970:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177155.578972:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1786177155.578975:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177155.578977:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177155.578979:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177155.578981:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177155.578983:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177155.578985:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177155.578986:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177155.578989:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.578995:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177155.578997:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579001:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.579002:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177155.579005:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177155.579007:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177155.579009:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177155.579011:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302051464
00000100:00000001:3.0:1786177155.579013:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302055117
00000100:00000001:3.0:1786177155.579014:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302051464
00000100:00000001:3.0:1786177155.579016:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302055118
00000100:00000001:3.0:1786177155.579018:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177155.579019:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.579021:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177155.579023:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177155/real 1786177155]  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177155.579029:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177155.579030:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177155.579031:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177155.579032:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177155.579033:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177155.579035:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177155.579037:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177155.579043:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579047:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177155.579048:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579049:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177155.579050:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177155.579052:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579056:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177155.579058:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579062:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177155.579064:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579065:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177155.579066:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579067:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177155.579069:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177155.579071:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177155.579074:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177155.579075:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177155.579078:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177155.579083:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177155.579085:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579087:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177155.579089:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41ad80 x1872934856343680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579093:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177155.579094:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177155.579095:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177155.579097:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177155.579098:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177155.579099:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177155.579100:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177155.579102:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:3.0:1786177155.579104:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177155.579105:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177155.579106:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41ad80.
00000100:00000001:3.0:1786177155.579108:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177155.579109:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579110:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177155.579112:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b480 x1872934856343808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579116:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177155.579117:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177155.579118:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177155.579119:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177155.579120:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177155.579121:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177155.579122:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177155.579124:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
02000000:00000001:3.0:1786177155.579126:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177155.579126:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177155.579128:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b480.
00000100:00000001:3.0:1786177155.579129:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177155.579130:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579131:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579134:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177155.579135:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.579137:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177155.579138:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177155.579139:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177155.579142:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579147:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177155.579148:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177155.579150:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177155.579151:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177155.579152:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177155.579154:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177155.579156:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177155.579157:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177155.579158:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177155.579160:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177155.579161:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177155.579178:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.579181:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177155.579183:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177155.579186:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177155.579189:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856343936, portal 4
00000100:00000001:3.0:1786177155.579190:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177155.579191:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761667328 : -114547947884288 : ffff97d1ba41b100)
00000100:00000040:3.0:1786177155.579196:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579205:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177155.579207:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177155.579208:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177155.579210:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856343936, offset 0
00000400:00000200:3.0:1786177155.579213:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177155.579215:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177155.579218:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177155.579220:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177155.579221:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177155.579223:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177155.579225:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177155.579227:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254858:254858:256:4294967295] 192.168.202.110@tcp LPNI seq info [254847:254847:8:4294967295]
00000400:00000200:3.0:1786177155.579230:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177155.579234:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177155.579237:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177155.579239:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177155.579242:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177155.579244:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786177155.579247:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177155.579248:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1786177155.579251:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177155.579253:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177155.579255:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177155.579257:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177155.579259:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177155.579261:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177155.579263:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177155.579265:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579270:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177155.579272:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579281:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.579282:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177155.579284:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177155.579287:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177155.579289:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177155.579291:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302051339
00000100:00000001:3.0:1786177155.579293:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302055117
00000100:00000001:3.0:1786177155.579294:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302051339
00000100:00000001:3.0:1786177155.579296:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302055118
00000100:00000001:3.0:1786177155.579297:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177155.579299:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.579300:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177155.579302:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177155/real 1786177155]  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177155.579307:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177155.579308:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177155.579309:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177155.579311:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177155.579312:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177155.579313:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177155.579315:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177155.579321:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579328:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177155.579330:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579332:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177155.579333:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177155.579337:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579345:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177155.579349:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579357:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177155.579360:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579361:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177155.579363:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579365:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177155.579367:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177155.579369:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177155.579373:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177155.579375:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177155.579379:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177155.579388:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856343936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177155.579391:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177155.579395:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579401:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177155.579403:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177155.579405:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177155.579407:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177155.579409:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177155.579412:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856344064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177155.579415:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177155.579418:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177155.579420:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177155.579421:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177155.579424:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177155.579426:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177155.579429:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177155.579432:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177155.579435:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177155.579438:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856344064, portal 4
00000100:00000001:3.0:1786177155.579440:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177155.579441:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761659264 : -114547947892352 : ffff97d1ba419180)
00000100:00000040:3.0:1786177155.579446:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579453:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177155.579455:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177155.579457:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177155.579460:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856344064, offset 0
00000400:00000200:3.0:1786177155.579463:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177155.579467:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177155.579470:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177155.579473:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177155.579476:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177155.579478:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177155.579480:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177155.579484:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254859:254859:256:4294967295] 192.168.202.110@tcp LPNI seq info [254848:254848:8:4294967295]
00000400:00000200:3.0:1786177155.579488:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177155.579494:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177155.579500:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177155.579503:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177155.579506:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177155.579508:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786177155.579531:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177155.579533:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1786177155.579536:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177155.579538:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177155.579540:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177155.579543:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177155.579545:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177155.579546:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177155.579548:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177155.579551:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579557:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177155.579559:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579564:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.579565:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177155.579567:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177155.579570:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177155.579572:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177155.579574:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302051348
00000100:00000001:3.0:1786177155.579576:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302055117
00000100:00000001:3.0:1786177155.579577:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302051348
00000100:00000001:3.0:1786177155.579579:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302055118
00000100:00000001:3.0:1786177155.579581:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177155.579583:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.579586:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177155.579589:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177155/real 1786177155]  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177155.579598:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177155.579600:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177155.579602:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177155.579605:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177155.579607:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177155.579610:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177155.579614:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177155.579622:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579629:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177155.579631:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579632:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177155.579634:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177155.579638:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579646:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177155.579649:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579654:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177155.579656:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579657:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177155.579658:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579659:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177155.579661:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177155.579662:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177155.579664:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177155.579666:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177155.579670:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177155.579677:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856344064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177155.579681:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579683:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177155.579687:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b100 x1872934856343936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579694:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177155.579696:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177155.579698:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177155.579701:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177155.579702:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177155.579704:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177155.579706:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177155.579708:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:3.0:1786177155.579711:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177155.579713:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177155.579714:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b100.
00000100:00000001:3.0:1786177155.579717:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177155.579719:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579721:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177155.579724:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419180 x1872934856344064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177155.579729:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177155.579730:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177155.579731:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177155.579733:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177155.579733:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177155.579734:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177155.579735:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177155.579736:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:3.0:1786177155.579737:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177155.579738:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177155.579739:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419180.
00000100:00000001:3.0:1786177155.579740:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177155.579741:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579742:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177155.579745:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177155.579746:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177155.579747:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177155.579748:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177156.152630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177156.152639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177156.152647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177156.152648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177156.152652:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177156.152653:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177156.152711:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177156.152718:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177156.152722:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177156.600603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177156.600609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177156.600612:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177156.600617:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177156.600620:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177156.600624:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177156.600626:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177156.600629:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177156.600632:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177157.176590:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177157.176595:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177157.176599:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177157.176600:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177157.176602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177157.176603:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177157.176646:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177157.176650:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177157.176652:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177157.624603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177157.624610:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177157.624613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177157.624618:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177157.624622:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177157.624626:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177157.624630:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177157.624633:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177157.624636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177158.200610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177158.200615:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177158.200618:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177158.200619:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177158.200622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177158.200623:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177158.200682:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177158.200685:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177158.200687:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177158.648614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177158.648620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177158.648622:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177158.648626:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177158.648628:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177158.648632:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177158.648634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177158.648636:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177158.648638:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177159.224630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177159.224638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177159.224644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177159.224645:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177159.224648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177159.224649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177159.224709:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177159.224713:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177159.224716:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177159.416619:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302055117, at 4302055168
00000100:00000040:1.0:1786177159.416619:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302055118, at 4302055168
00000100:00000010:1.0:1786177159.416632:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad378a0.
00000100:00000010:0.0:1786177159.416635:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37600.
00000100:00000001:2.0:1786177159.672565:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177159.672570:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177159.672572:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177159.672576:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177159.672579:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177159.672582:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177159.672584:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177159.672586:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177159.672589:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177160.248623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177160.248630:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177160.248636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177160.248637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177160.248640:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177160.248642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177160.248686:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177160.248689:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177160.248691:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786177160.696620:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786177160.696622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177160.696627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786177160.696630:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177160.696631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177160.696634:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:2.0:1786177160.696637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:0.0:1786177160.696639:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177160.696643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177160.696644:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786177160.696648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177160.696651:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177160.696653:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177160.696654:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7384
00000100:00000001:2.0:1786177160.696656:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177160.696666:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177160.696672:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177160.696674:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177160.696677:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696685:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177160.696687:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696689:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177160.696692:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177160.696694:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177160.696696:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177160.696698:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177160.696700:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177160.696701:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177160.696703:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696704:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177160.696705:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696706:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177160.696708:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177160.696710:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696712:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177160.696714:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177160.696717:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177160.696720:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177160.696721:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177160.696730:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177160.696739:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418a80.
00000020:00000040:0.0:1786177160.696743:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177160.696746:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177160.696748:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177160.696751:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177160.696754:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177160.696758:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177160.696765:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000100:00000001:0.0:1786177160.696770:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177160.696777:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177160.696784:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177160.696787:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177160.696791:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000606c34cb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177160.696817:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177160.696819:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177160.696823:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177160.696824:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177160.696825:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177160.696828:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177160.696831:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7384
00000400:00000200:0.0:1786177160.696834:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177160.696837:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177160.696839:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177160.696842:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696844:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177160.696845:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696847:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177160.696849:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177160.696851:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177160.696853:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177160.696856:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177160.696859:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1786177160.696861:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:3.0:1786177160.696861:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1786177160.696863:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177160.696865:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177160.696867:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:3.0:1786177160.696867:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177160.696869:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177160.696871:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696873:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177160.696875:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000040:0.0:1786177160.696876:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:3.0:1786177160.696876:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177160.696878:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696880:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.696880:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177160.696882:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177160.696882:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177160.696884:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177160.696887:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1786177160.696887:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856344192:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1786177160.696889:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177160.696890:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177160.696891:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:0.0:1786177160.696894:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1786177160.696894:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:0.0:1786177160.696896:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.696897:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177160.696898:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.696899:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:0.0:1786177160.696902:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418e00.
00000020:00000040:0.0:1786177160.696904:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000010:3.0:1786177160.696904:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177160.696906:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177160.696906:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177160.696907:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177160.696909:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177160.696910:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177160.696911:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177160.696911:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:0.0:1786177160.696915:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000400:00000010:3.0:1786177160.696915:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000001:0.0:1786177160.696918:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177160.696921:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00100000:0.0:1786177160.696922:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177160.696928:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:3.0:1786177160.696928:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856344192, portal 25
00000100:00000001:0.0:1786177160.696930:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.696930:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177160.696931:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761657472 : -114547947894144 : ffff97d1ba418a80)
00000100:00000040:0.0:1786177160.696934:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003428892b] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177160.696936:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177160.696942:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.696944:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:0.0:1786177160.696945:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1786177160.696947:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177160.696948:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177160.696949:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:3.0:1786177160.696949:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00080000:0.0:1786177160.696951:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177160.696953:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:3.0:1786177160.696953:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856344192, offset 0
00000100:00080000:0.0:1786177160.696955:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7384
00000400:00000200:0.0:1786177160.696958:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177160.696959:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177160.696961:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177160.696962:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177160.696964:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696966:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177160.696966:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177160.696968:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696970:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177160.696972:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:3.0:1786177160.696972:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000040:0.0:1786177160.696973:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177160.696974:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177160.696976:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:3.0:1786177160.696976:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:0.0:1786177160.696978:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177160.696979:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177160.696980:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:0.0:1786177160.696982:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696983:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177160.696984:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1786177160.696985:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177160.696986:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696987:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177160.696989:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:3.0:1786177160.696990:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254860:254860:256:4294967295] 192.168.202.110@tcp LPNI seq info [254849:254849:8:4294967295]
00000100:00080000:0.0:1786177160.696991:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177160.696993:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177160.696995:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1786177160.696995:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786177160.696997:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177160.696998:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177160.697000:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177160.697002:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:3.0:1786177160.697003:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000008:00000001:0.0:1786177160.697004:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177160.697005:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177160.697009:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419180.
00000800:00000200:3.0:1786177160.697009:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:0.0:1786177160.697011:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177160.697013:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000010:3.0:1786177160.697013:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
02000000:00000001:0.0:1786177160.697014:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177160.697016:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177160.697017:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177160.697018:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177160.697019:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:0.0:1786177160.697021:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
00000800:00000200:3.0:1786177160.697022:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:0.0:1786177160.697024:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177160.697028:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177160.697031:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000100:00100000:0.0:1786177160.697035:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:3.0:1786177160.697035:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177160.697038:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00000001:0.0:1786177160.697041:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177160.697043:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177160.697044:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1786177160.697046:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005a89d8ab] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177160.697048:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177160.697052:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000100:00000001:0.0:1786177160.697053:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177160.697054:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00080000:0.0:1786177160.697055:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177160.697058:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177160.697058:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1786177160.697059:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177160.697061:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1786177160.697061:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177160.697063:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177160.697066:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7384
00000400:00000200:0.0:1786177160.697069:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177160.697070:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000400:00000200:0.0:1786177160.697071:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177160.697073:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177160.697073:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177160.697076:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.697078:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177160.697080:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177160.697080:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177160.697082:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177160.697082:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:0.0:1786177160.697084:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177160.697085:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177160.697086:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177160.697087:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177160.697088:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177160.697089:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177160.697091:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000200:3.0:1786177160.697091:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177160.697094:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177160.697094:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302060237
00000100:00000001:0.0:1786177160.697096:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177160.697098:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.697099:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000010:3.0:1786177160.697099:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37300.
00000020:00000040:0.0:1786177160.697101:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177160.697103:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177160.697104:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786177160.697106:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177160.697107:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177160.697109:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177160.697109:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302060237
02000000:00000001:0.0:1786177160.697111:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177160.697112:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37d20.
00000100:00000001:0.0:1786177160.697113:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177160.697126:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177160.697129:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.697129:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786177160.697130:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.697132:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:0.0:1786177160.697134:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b100.
00000100:00000001:3.0:1786177160.697134:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177160.697136:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177160.697136:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000001:0.0:1786177160.697138:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177160.697139:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177160.697140:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:3.0:1786177160.697141:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177160/real 1786177160]  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177160.697142:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177160.697143:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177160.697147:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000100:00000001:0.0:1786177160.697149:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177160.697149:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177160.697150:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177160.697152:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00100000:0.0:1786177160.697153:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177160.697155:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177160.697156:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177160.697159:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177160.697162:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177160.697167:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177160.697169:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177160.697171:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177160.697172:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000fa632fb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697178:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177160.697179:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177160.697180:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.697181:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:0.0:1786177160.697182:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177160.697183:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177160.697185:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177160.697187:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1786177160.697187:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177160.697189:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177160.697191:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177160.697193:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7384
00000100:00000001:3.0:1786177160.697195:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1786177160.697197:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177160.697199:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1786177160.697199:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177160.697201:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177160.697204:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.697206:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177160.697206:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:0.0:1786177160.697208:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.697209:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177160.697209:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.697210:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000001:0.0:1786177160.697212:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177160.697212:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1786177160.697213:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177160.697214:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177160.697214:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000040:0.0:1786177160.697216:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:3.0:1786177160.697216:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:0.0:1786177160.697219:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00080000:3.0:1786177160.697219:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000020:00000001:0.0:1786177160.697221:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00080000:3.0:1786177160.697223:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000040:0.0:1786177160.697224:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.697225:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177160.697225:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177160.697227:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177160.697229:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:3.0:1786177160.697229:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177160.697231:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177160.697233:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177160.697235:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177160.697237:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:3.0:1786177160.697238:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856344192:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1786177160.697239:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177160.697241:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177160.697242:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177160.697242:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:0.0:1786177160.697244:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177160.697244:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000008:00000001:0.0:1786177160.697246:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177160.697247:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418a80 x1872934856344192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177160.697248:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177160.697251:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b480.
00000020:00000040:0.0:1786177160.697253:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177160.697253:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177160.697255:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177160.697256:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.697256:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:0.0:1786177160.697258:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:3.0:1786177160.697258:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177160.697259:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177160.697260:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.697261:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000010:0.0:1786177160.697263:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000020:00000001:3.0:1786177160.697263:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177160.697264:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:0.0:1786177160.697266:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177160.697266:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177160.697268:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
00000100:00100000:0.0:1786177160.697269:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1786177160.697272:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177160.697273:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000001:0.0:1786177160.697275:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000010:3.0:1786177160.697275:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418a80.
00000100:00000001:0.0:1786177160.697277:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.697278:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177160.697280:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177160.697281:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001215e9e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697282:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177160.697287:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.697288:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177160.697289:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177160.697291:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7394)
00000100:00000001:3.0:1786177160.697291:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177160.697292:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177160.697293:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177160.697296:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697309:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177160.697310:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.697312:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177160.697314:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177160.697316:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177160.697318:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856344320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177160.697321:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177160.697323:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177160.697325:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.697326:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177160.697329:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177160.697330:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177160.697334:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177160.697337:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177160.697340:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177160.697343:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856344320, portal 10
00000100:00000001:3.0:1786177160.697345:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177160.697346:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761658368 : -114547947893248 : ffff97d1ba418e00)
00000100:00000040:3.0:1786177160.697352:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697358:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177160.697360:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177160.697362:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177160.697365:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856344320, offset 0
00000400:00000200:3.0:1786177160.697369:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177160.697373:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177160.697375:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177160.697377:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177160.697379:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177160.697380:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177160.697382:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177160.697384:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254861:254861:256:4294967295] 192.168.202.110@tcp LPNI seq info [254850:254850:8:4294967295]
00000400:00000200:3.0:1786177160.697386:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177160.697390:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177160.697393:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177160.697395:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177160.697398:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177160.697399:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786177160.697402:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177160.697404:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1786177160.697406:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177160.697407:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177160.697409:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177160.697411:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177160.697413:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177160.697414:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177160.697415:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177160.697417:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697422:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177160.697424:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697430:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.697431:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177160.697434:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177160.697438:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177160.697441:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177160.697442:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302056457
00000100:00000001:3.0:1786177160.697445:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302060237
00000100:00000001:3.0:1786177160.697447:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302056457
00000100:00000001:3.0:1786177160.697449:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302060237
00000100:00000001:3.0:1786177160.697451:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177160.697453:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.697454:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177160.697458:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177160/real 1786177160]  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177160.697465:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177160.697466:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177160.697468:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177160.697470:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177160.697472:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177160.697474:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177160.697477:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177160.697486:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697492:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177160.697494:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.697496:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177160.697497:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177160.697500:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697507:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177160.697531:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697538:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177160.697540:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.697541:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177160.697543:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.697544:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177160.697546:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177160.697548:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177160.697552:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177160.697554:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177160.697559:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177160.697566:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856344320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177160.697570:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177160.697573:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697577:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177160.697579:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.697589:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177160.697591:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177160.697593:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177160.697596:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856344448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177160.697599:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177160.697601:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177160.697602:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.697604:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177160.697608:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177160.697610:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177160.697612:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177160.697614:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177160.697616:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177160.697618:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856344448, portal 10
00000100:00000001:3.0:1786177160.697619:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177160.697620:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761659264 : -114547947892352 : ffff97d1ba419180)
00000100:00000040:3.0:1786177160.697624:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697628:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177160.697629:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177160.697631:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177160.697632:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856344448, offset 0
00000400:00000200:3.0:1786177160.697634:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177160.697637:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177160.697639:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177160.697641:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177160.697643:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177160.697644:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177160.697645:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177160.697647:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254862:254862:256:4294967295] 192.168.202.110@tcp LPNI seq info [254851:254851:8:4294967295]
00000400:00000200:3.0:1786177160.697650:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177160.697654:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177160.697656:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177160.697658:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177160.697660:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177160.697662:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786177160.697664:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177160.697666:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1786177160.697668:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177160.697669:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177160.697671:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177160.697673:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177160.697675:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177160.697676:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177160.697678:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177160.697680:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697685:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177160.697687:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697693:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.697695:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177160.697698:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177160.697700:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177160.697703:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177160.697705:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302056583
00000100:00000001:3.0:1786177160.697707:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302060237
00000100:00000001:3.0:1786177160.697709:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302056583
00000100:00000001:3.0:1786177160.697721:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302060237
00000100:00000001:3.0:1786177160.697724:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177160.697726:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.697727:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177160.697731:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177160/real 1786177160]  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177160.697736:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177160.697738:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177160.697739:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177160.697740:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177160.697741:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177160.697743:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177160.697745:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177160.697750:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697753:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177160.697754:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.697755:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177160.697755:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177160.697757:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697761:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177160.697763:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697768:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177160.697769:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.697770:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177160.697771:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.697772:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177160.697773:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177160.697774:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177160.697776:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177160.697777:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177160.697779:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177160.697784:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856344448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177160.697786:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177160.697787:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697791:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177160.697791:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.697793:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177160.697794:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177160.697795:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177160.697796:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856344576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177160.697798:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177160.697799:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177160.697800:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.697801:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177160.697805:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177160.697806:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177160.697810:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177160.697812:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177160.697815:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177160.697817:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856344576, portal 4
00000100:00000001:3.0:1786177160.697819:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177160.697821:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761667328 : -114547947884288 : ffff97d1ba41b100)
00000100:00000040:3.0:1786177160.697826:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697832:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177160.697834:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177160.697836:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177160.697838:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856344576, offset 0
00000400:00000200:3.0:1786177160.697842:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177160.697845:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177160.697848:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177160.697850:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177160.697851:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177160.697853:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177160.697854:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177160.697856:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254863:254863:256:4294967295] 192.168.202.110@tcp LPNI seq info [254852:254852:8:4294967295]
00000400:00000200:3.0:1786177160.697858:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177160.697863:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177160.697865:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177160.697868:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177160.697870:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177160.697871:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786177160.697874:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177160.697875:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1786177160.697877:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177160.697879:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177160.697880:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177160.697882:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177160.697884:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177160.697886:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177160.697887:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177160.697889:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697894:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177160.697896:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697899:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.697900:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177160.697902:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177160.697904:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177160.697905:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177160.697907:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302056458
00000100:00000001:3.0:1786177160.697908:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302060237
00000100:00000001:3.0:1786177160.697909:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302056458
00000100:00000001:3.0:1786177160.697911:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302060237
00000100:00000001:3.0:1786177160.697912:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177160.697913:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.697914:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177160.697916:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177160/real 1786177160]  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177160.697921:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177160.697921:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177160.697923:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177160.697924:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177160.697925:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177160.697926:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177160.697928:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177160.697932:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697936:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177160.697937:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.697938:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177160.697939:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177160.697941:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697944:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177160.697946:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697950:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177160.697951:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.697952:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177160.697952:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.697953:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177160.697954:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177160.697955:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177160.697957:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177160.697958:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177160.697960:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177160.697964:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856344576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177160.697967:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177160.697968:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.697972:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177160.697972:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.697974:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177160.697975:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177160.697976:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177160.697978:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856344704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177160.697980:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177160.697981:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177160.697981:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177160.697982:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177160.697984:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177160.697985:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177160.697987:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177160.697988:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177160.697990:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177160.697992:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856344704, portal 4
00000100:00000001:3.0:1786177160.697993:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177160.697994:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761668224 : -114547947883392 : ffff97d1ba41b480)
00000100:00000040:3.0:1786177160.697997:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.698001:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177160.698002:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177160.698003:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177160.698005:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856344704, offset 0
00000400:00000200:3.0:1786177160.698007:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177160.698008:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177160.698010:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177160.698012:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177160.698013:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177160.698014:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177160.698015:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177160.698017:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254864:254864:256:4294967295] 192.168.202.110@tcp LPNI seq info [254853:254853:8:4294967295]
00000400:00000200:3.0:1786177160.698020:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177160.698023:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177160.698026:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177160.698027:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177160.698029:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177160.698031:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786177160.698033:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177160.698035:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1786177160.698037:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177160.698038:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177160.698039:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177160.698041:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177160.698043:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177160.698044:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177160.698045:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177160.698047:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.698051:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177160.698053:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.698057:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.698059:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177160.698061:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177160.698064:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177160.698067:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177160.698068:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302056466
00000100:00000001:3.0:1786177160.698070:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302060237
00000100:00000001:3.0:1786177160.698072:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302056466
00000100:00000001:3.0:1786177160.698074:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302060237
00000100:00000001:3.0:1786177160.698076:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177160.698078:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.698080:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177160.698084:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177160/real 1786177160]  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177160.698091:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177160.698092:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177160.698093:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177160.698095:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177160.698097:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177160.698098:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177160.698101:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177160.698109:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.698127:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177160.698128:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.698130:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177160.698131:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177160.698135:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.698140:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177160.698144:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.698150:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177160.698153:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.698154:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177160.698155:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.698157:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177160.698158:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177160.698161:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177160.698163:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177160.698164:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177160.698167:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177160.698172:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856344704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177160.698174:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.698176:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177160.698177:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418e00 x1872934856344320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.698181:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177160.698182:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177160.698184:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177160.698185:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177160.698186:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177160.698187:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177160.698188:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177160.698189:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:3.0:1786177160.698191:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177160.698192:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177160.698193:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418e00.
00000100:00000001:3.0:1786177160.698194:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177160.698195:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.698196:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177160.698198:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419180 x1872934856344448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.698202:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177160.698202:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177160.698203:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177160.698205:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177160.698205:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177160.698206:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177160.698207:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177160.698207:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
02000000:00000001:3.0:1786177160.698209:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177160.698209:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177160.698210:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419180.
00000100:00000001:3.0:1786177160.698212:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177160.698212:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.698213:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177160.698215:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b100 x1872934856344576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.698218:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177160.698219:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177160.698220:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177160.698221:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177160.698221:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177160.698222:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177160.698223:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177160.698223:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:3.0:1786177160.698224:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177160.698225:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177160.698226:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b100.
00000100:00000001:3.0:1786177160.698227:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177160.698228:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.698228:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177160.698230:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b480 x1872934856344704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177160.698233:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177160.698233:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177160.698234:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177160.698235:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177160.698236:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177160.698236:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177160.698237:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177160.698238:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:3.0:1786177160.698239:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177160.698239:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177160.698240:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b480.
00000100:00000001:3.0:1786177160.698241:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177160.698242:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.698243:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177160.698245:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177160.698246:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177160.698248:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177160.698249:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177161.272591:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177161.272597:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177161.272598:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177161.272600:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177161.272604:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177161.272606:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177161.272607:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177161.272609:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177161.272611:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177161.272612:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177161.272613:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177161.272614:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177161.272615:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177161.272616:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177161.272617:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177161.272618:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177161.272620:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177161.272621:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177161.272661:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177161.272664:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177161.272667:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177161.720590:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177161.720598:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177161.720601:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177161.720606:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177161.720611:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177161.720614:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177161.720618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177161.720620:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177161.720623:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177162.296588:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177162.296594:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177162.296600:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177162.296601:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177162.296604:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177162.296606:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177162.296664:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177162.296669:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177162.296672:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177162.744617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177162.744625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177162.744628:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177162.744633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177162.744637:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177162.744641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177162.744644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177162.744646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177162.744649:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177163.320597:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177163.320604:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177163.320609:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177163.320610:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177163.320613:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177163.320616:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177163.320618:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177163.320621:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177163.320624:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177163.320626:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177163.320628:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177163.320630:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177163.320679:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177163.320683:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177163.320685:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177163.768607:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177163.768614:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177163.768615:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177163.768620:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177163.768624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177163.768627:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177163.768630:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177163.768632:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177163.768635:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177164.344599:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177164.344604:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177164.344608:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177164.344609:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177164.344611:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177164.344612:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177164.344661:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177164.344664:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177164.344666:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177164.536572:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302060237, at 4302060288
00000100:00000010:0.0:1786177164.536587:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37300.
00000100:00000040:1.0:1786177164.536599:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302060237, at 4302060288
00000100:00000010:1.0:1786177164.536606:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37d20.
00000100:00000001:2.0:1786177164.792580:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177164.792585:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177164.792587:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177164.792591:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177164.792593:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177164.792597:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177164.792599:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177164.792601:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177164.792603:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177165.368661:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177165.368668:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177165.368673:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177165.368675:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177165.368677:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177165.368678:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177165.368720:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177165.368725:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177165.368728:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786177165.816623:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177165.816632:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177165.816634:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177165.816635:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177165.816640:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177165.816641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177165.816645:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.816647:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:0.0:1786177165.816650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177165.816655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1786177165.816656:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7389
00000100:00000001:0.0:1786177165.816661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177165.816665:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000400:00000200:2.0:1786177165.816667:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177165.816669:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177165.816674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177165.816676:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177165.816680:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177165.816684:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.816687:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177165.816690:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.816692:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177165.816696:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177165.816698:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177165.816702:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177165.816705:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177165.816708:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177165.816710:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177165.816713:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.816715:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177165.816717:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.816719:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177165.816721:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177165.816725:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177165.816727:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177165.816730:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177165.816733:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177165.816752:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.816754:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177165.816757:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177165.816765:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786177165.816769:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177165.816772:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177165.816774:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.816777:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177165.816780:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177165.816784:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177165.816796:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786177165.816803:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177165.816818:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177165.816828:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177165.816833:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177165.816837:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177165.816859:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177165.816862:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177165.816865:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177165.816867:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177165.816868:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177165.816871:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177165.816873:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7389
00000400:00000200:2.0:1786177165.816877:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177165.816880:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177165.816882:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177165.816885:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.816888:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177165.816891:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.816893:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177165.816896:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177165.816897:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1786177165.816898:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:3.0:1786177165.816901:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1786177165.816902:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177165.816904:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1786177165.816906:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177165.816909:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177165.816911:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177165.816912:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177165.816915:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.816920:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177165.816924:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.816927:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177165.816930:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1786177165.816931:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:3.0:1786177165.816933:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177165.816935:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1786177165.816937:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1786177165.816938:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177165.816941:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177165.816941:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177165.816944:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177165.816945:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1786177165.816948:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856344832:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1786177165.816949:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.816953:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177165.816954:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:2.0:1786177165.816956:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1786177165.816957:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:2.0:1786177165.816960:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177165.816961:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.816964:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177165.816965:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:2.0:1786177165.816971:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
02000000:00000010:3.0:1786177165.816972:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
00000020:00000040:2.0:1786177165.816973:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177165.816977:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177165.816978:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177165.816980:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.816982:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177165.816983:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1786177165.816984:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177165.816985:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177165.816989:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
02000000:00000010:2.0:1786177165.816991:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000001:2.0:1786177165.816995:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177165.816996:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00100000:2.0:1786177165.817001:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1786177165.817002:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856344832, portal 25
00000100:00000001:3.0:1786177165.817007:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177165.817009:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454181632 : -114548255369984 : ffff97d1a7edd500)
00000100:00000001:2.0:1786177165.817015:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177165.817018:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177165.817020:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177165.817024:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817037:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786177165.817041:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177165.817041:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786177165.817045:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1786177165.817046:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177165.817050:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856344832, offset 0
00000100:00080000:2.0:1786177165.817051:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177165.817054:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177165.817056:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177165.817058:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177165.817059:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177165.817077:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177165.817082:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177165.817085:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177165.817088:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177165.817091:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177165.817092:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177165.817098:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254865:254865:256:4294967295] 192.168.202.110@tcp LPNI seq info [254854:254854:8:4294967295]
00000100:00080000:2.0:1786177165.817099:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7389
00000400:00000200:3.0:1786177165.817105:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177165.817110:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177165.817117:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1786177165.817118:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1786177165.817122:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1786177165.817127:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177165.817130:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.817133:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:3.0:1786177165.817133:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000040:2.0:1786177165.817138:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.817140:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.0:1786177165.817144:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786177165.817145:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177165.817147:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177165.817150:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:3.0:1786177165.817150:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000040:2.0:1786177165.817154:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000800:00000200:3.1:1786177165.817158:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786177165.817160:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:3.1:1786177165.817163:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:2.0:1786177165.817165:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000100:3.0:1786177165.817169:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177165.817172:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000010:3.0:1786177165.817173:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00000001:2.0:1786177165.817176:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177165.817179:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177165.817180:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1786177165.817181:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177165.817184:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:3.0:1786177165.817188:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:2.0:1786177165.817190:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177165.817192:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177165.817193:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
02000000:00000001:2.0:1786177165.817194:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1786177165.817196:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
02000000:00000001:2.0:1786177165.817198:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177165.817200:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1786177165.817201:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.817204:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177165.817206:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177165.817207:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.817208:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177165.817210:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000100:00000200:3.0:1786177165.817212:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177165.817213:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177165.817218:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177165.817221:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.817225:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177165.817227:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177165.817229:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.817229:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000010:2.0:1786177165.817235:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000040:3.0:1786177165.817235:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177165.817239:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.817245:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177165.817247:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177165.817248:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177165.817254:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177165.817254:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.817258:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177165.817262:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786177165.817266:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817267:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302065357
00000100:00000010:3.0:1786177165.817274:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad371e0.
00000100:00000001:2.0:1786177165.817275:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177165.817278:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177165.817280:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177165.817281:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177165.817282:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177165.817285:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177165.817288:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177165.817289:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302065357
00000100:00080000:2.0:1786177165.817291:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7389
00000100:00000010:3.0:1786177165.817293:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37b40.
00000100:00000001:3.0:1786177165.817296:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000400:00000200:2.0:1786177165.817297:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177165.817299:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1786177165.817302:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177165.817303:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177165.817304:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177165.817307:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1786177165.817309:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.817311:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:3.0:1786177165.817314:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177165/real 1786177165]  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177165.817315:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.817320:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000001:3.0:1786177165.817326:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177165.817327:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177165.817329:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177165.817330:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000020:00000001:2.0:1786177165.817333:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000010:3.0:1786177165.817334:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000020:00000040:2.0:1786177165.817336:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:3.0:1786177165.817337:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:2.0:1786177165.817340:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177165.817344:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000200:3.0:1786177165.817344:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177165.817347:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177165.817352:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786177165.817358:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.817360:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1786177165.817362:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177165.817364:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.817368:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177165.817371:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177165.817377:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177165.817377:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177165.817379:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.817380:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786177165.817382:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177165.817383:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:2.0:1786177165.817385:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177165.817386:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786177165.817392:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177165.817394:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177165.817395:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177165.817398:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177165.817403:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.817407:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.817409:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:2.0:1786177165.817411:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000100:00000040:3.0:1786177165.817414:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177165.817415:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177165.817420:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177165.817422:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.817426:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177165.817428:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:2.0:1786177165.817429:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177165.817431:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.817434:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:2.0:1786177165.817436:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:3.0:1786177165.817436:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177165.817438:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177165.817440:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.817441:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177165.817445:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00100000:2.0:1786177165.817448:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00080000:3.0:1786177165.817449:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177165.817456:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177165.817460:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1786177165.817465:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1786177165.817468:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177165.817469:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177165.817476:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177165.817484:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856344832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177165.817489:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.817490:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177165.817492:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177165.817494:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:2.0:1786177165.817496:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000040:3.0:1786177165.817499:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd500 x1872934856344832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817505:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177165.817507:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177165.817529:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
00000100:00000001:2.0:1786177165.817530:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177165.817533:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:3.0:1786177165.817533:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177165.817536:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177165.817538:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177165.817539:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177165.817541:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:3.0:1786177165.817543:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177165.817545:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177165.817546:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd500.
00000100:00000001:3.0:1786177165.817548:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177165.817551:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177165.817552:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.817555:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177165.817557:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7389
00000400:00000200:2.0:1786177165.817562:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177165.817562:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177165.817565:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177165.817566:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177165.817569:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:2.0:1786177165.817570:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177165.817572:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1786177165.817574:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177165.817575:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786177165.817579:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177165.817581:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177165.817583:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.817587:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177165.817591:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177165.817593:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177165.817596:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177165.817597:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1786177165.817599:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:3.0:1786177165.817600:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177165.817605:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
02000000:00000001:3.0:1786177165.817606:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177165.817608:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:2.0:1786177165.817610:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
02000000:00000001:3.0:1786177165.817611:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177165.817616:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00100000:3.0:1786177165.817617:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856344960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177165.817622:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177165.817622:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177165.817625:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177165.817627:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177165.817628:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177165.817629:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786177165.817632:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000010:3.0:1786177165.817634:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
00000020:00000040:2.0:1786177165.817635:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
02000000:00000001:3.0:1786177165.817636:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177165.817640:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1786177165.817641:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177165.817643:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177165.817644:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
02000000:00000001:2.0:1786177165.817646:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1786177165.817650:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
02000000:00000001:2.0:1786177165.817651:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177165.817655:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177165.817655:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856344960, portal 10
00000100:00000001:2.0:1786177165.817660:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177165.817660:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000008:00100000:2.0:1786177165.817662:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177165.817663:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178048 : -114548255373568 : ffff97d1a7edc700)
00000008:00000001:2.0:1786177165.817667:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177165.817670:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177165.817673:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786177165.817675:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786177165.817679:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177165.817684:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177165.817684:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:2.0:1786177165.817686:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177165.817688:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177165.817689:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177165.817692:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1786177165.817693:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
02000000:00000001:2.0:1786177165.817696:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177165.817696:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856344960, offset 0
02000000:00000010:2.0:1786177165.817703:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000400:00000200:3.0:1786177165.817703:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177165.817709:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177165.817709:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1786177165.817716:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177165.817716:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177165.817719:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177165.817724:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177165.817729:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1786177165.817730:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177165.817732:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1786177165.817734:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177165.817737:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254866:254866:256:4294967295] 192.168.202.110@tcp LPNI seq info [254855:254855:8:4294967295]
00000100:00000040:2.0:1786177165.817741:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177165.817745:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177165.817756:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177165.817757:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1786177165.817762:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7399)
00000800:00000200:3.0:1786177165.817762:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177165.817768:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177165.817771:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177165.817772:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786177165.817777:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177165.817778:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1786177165.817781:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177165.817782:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177165.817784:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177165.817786:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177165.817787:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177165.817788:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177165.817790:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177165.817792:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817797:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177165.817798:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817802:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.817803:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177165.817805:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177165.817808:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177165.817810:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177165.817811:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302061578
00000100:00000001:3.0:1786177165.817813:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302065357
00000100:00000001:3.0:1786177165.817814:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302061578
00000100:00000001:3.0:1786177165.817817:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302065357
00000100:00000001:3.0:1786177165.817818:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177165.817819:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.817820:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177165.817822:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177165/real 1786177165]  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177165.817826:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177165.817828:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177165.817828:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177165.817830:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177165.817831:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177165.817832:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177165.817833:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177165.817838:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817841:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177165.817842:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.817843:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177165.817844:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177165.817846:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817849:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177165.817851:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817854:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177165.817856:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.817856:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177165.817857:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.817859:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177165.817860:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177165.817862:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177165.817863:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177165.817865:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177165.817867:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177165.817871:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856344960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177165.817873:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177165.817874:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817878:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177165.817879:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177165.817880:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177165.817881:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177165.817882:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177165.817884:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177165.817886:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177165.817887:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177165.817887:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177165.817888:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177165.817890:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177165.817891:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177165.817893:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177165.817894:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177165.817896:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177165.817897:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856345088, portal 10
00000100:00000001:3.0:1786177165.817899:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177165.817899:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454180736 : -114548255370880 : ffff97d1a7edd180)
00000100:00000040:3.0:1786177165.817902:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817906:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177165.817908:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177165.817909:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177165.817910:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856345088, offset 0
00000400:00000200:3.0:1786177165.817912:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177165.817914:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177165.817915:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177165.817917:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177165.817918:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177165.817919:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177165.817921:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177165.817923:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254867:254867:256:4294967295] 192.168.202.110@tcp LPNI seq info [254856:254856:8:4294967295]
00000400:00000200:3.0:1786177165.817925:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177165.817929:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177165.817931:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177165.817933:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177165.817935:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177165.817937:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786177165.817939:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177165.817941:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1786177165.817943:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177165.817944:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177165.817945:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177165.817947:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177165.817948:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177165.817949:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177165.817951:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177165.817952:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817957:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177165.817958:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817966:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.817966:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177165.817968:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177165.817969:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177165.817971:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177165.817972:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302061703
00000100:00000001:3.0:1786177165.817974:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302065357
00000100:00000001:3.0:1786177165.817975:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302061703
00000100:00000001:3.0:1786177165.817976:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302065357
00000100:00000001:3.0:1786177165.817978:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177165.817979:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.817980:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177165.817981:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177165/real 1786177165]  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177165.817985:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177165.817986:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177165.817987:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177165.817988:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177165.817988:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177165.817989:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177165.817991:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177165.817995:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.817999:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177165.817999:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818000:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177165.818001:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177165.818003:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818007:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177165.818009:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818013:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177165.818014:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818014:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177165.818015:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818016:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177165.818017:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177165.818018:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177165.818020:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177165.818021:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177165.818023:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177165.818027:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177165.818029:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177165.818030:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818034:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177165.818034:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177165.818035:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177165.818036:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177165.818037:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177165.818039:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177165.818040:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177165.818041:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177165.818042:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177165.818043:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177165.818045:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177165.818046:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177165.818048:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177165.818049:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177165.818051:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177165.818053:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856345216, portal 4
00000100:00000001:3.0:1786177165.818054:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177165.818054:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187904 : -114548255363712 : ffff97d1a7eded80)
00000100:00000040:3.0:1786177165.818058:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818061:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177165.818062:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177165.818063:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177165.818079:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856345216, offset 0
00000400:00000200:3.0:1786177165.818082:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177165.818085:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177165.818088:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177165.818090:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177165.818092:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177165.818094:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177165.818096:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177165.818098:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254868:254868:256:4294967295] 192.168.202.110@tcp LPNI seq info [254857:254857:8:4294967295]
00000400:00000200:3.0:1786177165.818101:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177165.818104:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177165.818107:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177165.818109:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177165.818111:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177165.818113:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786177165.818115:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177165.818117:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1786177165.818119:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177165.818121:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177165.818122:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177165.818124:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177165.818126:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177165.818127:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177165.818128:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177165.818131:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818135:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177165.818137:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818141:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.818142:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177165.818143:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177165.818146:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177165.818147:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177165.818148:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302061578
00000100:00000001:3.0:1786177165.818150:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302065357
00000100:00000001:3.0:1786177165.818151:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302061578
00000100:00000001:3.0:1786177165.818152:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302065357
00000100:00000001:3.0:1786177165.818153:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177165.818154:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.818155:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177165.818157:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177165/real 1786177165]  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177165.818161:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177165.818162:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177165.818163:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177165.818165:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177165.818166:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177165.818167:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177165.818169:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177165.818173:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818177:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177165.818177:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818178:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177165.818179:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177165.818181:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818184:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177165.818186:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818190:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177165.818191:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818192:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177165.818193:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818194:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177165.818195:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177165.818197:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177165.818198:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177165.818199:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177165.818201:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177165.818205:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177165.818207:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818208:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177165.818210:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc700 x1872934856344960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818214:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177165.818215:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177165.818216:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177165.818217:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177165.818218:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177165.818219:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177165.818220:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177165.818221:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3400.
02000000:00000001:3.0:1786177165.818223:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177165.818224:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177165.818225:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc700.
00000100:00000001:3.0:1786177165.818226:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177165.818227:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818228:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177165.818230:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd180 x1872934856345088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818233:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177165.818234:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177165.818235:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177165.818236:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177165.818237:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177165.818238:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177165.818239:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177165.818240:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:3.0:1786177165.818241:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177165.818241:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177165.818242:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd180.
00000100:00000001:3.0:1786177165.818243:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177165.818243:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818244:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177165.818246:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eded80 x1872934856345216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818249:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177165.818250:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177165.818251:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177165.818252:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177165.818253:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177165.818254:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177165.818255:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177165.818256:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:3.0:1786177165.818257:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177165.818258:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177165.818258:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eded80.
00000100:00000001:3.0:1786177165.818260:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177165.818261:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818262:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818263:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177165.818264:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.818265:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177165.818266:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177165.818267:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177165.818268:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818272:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177165.818272:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177165.818274:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177165.818275:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177165.818276:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177165.818277:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345344:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177165.818280:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177165.818281:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177165.818281:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177165.818282:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177165.818284:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177165.818285:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177165.818287:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177165.818288:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177165.818290:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177165.818291:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856345344, portal 4
00000100:00000001:3.0:1786177165.818293:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177165.818294:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454181632 : -114548255369984 : ffff97d1a7edd500)
00000100:00000040:3.0:1786177165.818297:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818301:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177165.818302:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177165.818304:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177165.818305:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856345344, offset 0
00000400:00000200:3.0:1786177165.818307:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177165.818309:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177165.818310:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177165.818312:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177165.818313:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177165.818315:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177165.818316:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177165.818318:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254869:254869:256:4294967295] 192.168.202.110@tcp LPNI seq info [254858:254858:8:4294967295]
00000400:00000200:3.0:1786177165.818320:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177165.818323:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177165.818326:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177165.818327:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177165.818329:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177165.818331:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786177165.818333:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177165.818334:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1786177165.818336:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177165.818338:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177165.818339:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177165.818341:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177165.818343:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177165.818343:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177165.818345:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177165.818347:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818354:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177165.818356:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818360:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.818360:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177165.818362:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177165.818363:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177165.818365:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177165.818366:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302061586
00000100:00000001:3.0:1786177165.818367:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302065357
00000100:00000001:3.0:1786177165.818368:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302061586
00000100:00000001:3.0:1786177165.818370:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302065357
00000100:00000001:3.0:1786177165.818371:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177165.818372:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.818373:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177165.818375:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177165/real 1786177165]  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177165.818378:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177165.818379:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177165.818380:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177165.818381:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177165.818382:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177165.818383:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177165.818385:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177165.818389:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818392:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177165.818394:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818395:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177165.818396:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177165.818398:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818401:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177165.818403:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818406:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177165.818407:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818408:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177165.818409:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818410:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177165.818411:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177165.818412:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177165.818413:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177165.818415:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177165.818416:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177165.818420:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345344:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177165.818422:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818423:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177165.818425:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd500 x1872934856345344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177165.818428:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177165.818429:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177165.818430:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177165.818431:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177165.818432:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177165.818433:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177165.818434:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177165.818435:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:3.0:1786177165.818436:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177165.818437:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177165.818438:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd500.
00000100:00000001:3.0:1786177165.818439:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177165.818439:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818440:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177165.818442:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177165.818442:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177165.818443:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177165.818444:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177166.392617:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177166.392623:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177166.392628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177166.392630:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177166.392633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177166.392635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177166.392678:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177166.392681:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177166.392683:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177166.840578:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177166.840586:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177166.840588:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177166.840592:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177166.840595:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177166.840598:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177166.840600:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177166.840602:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177166.840605:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177167.416636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177167.416642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177167.416647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177167.416649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177167.416652:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177167.416654:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177167.416713:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177167.416717:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177167.416720:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177167.864626:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177167.864633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177167.864635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177167.864640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177167.864642:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177167.864646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177167.864648:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177167.864650:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177167.864652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177168.440617:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177168.440625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177168.440632:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177168.440633:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177168.440637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177168.440638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177168.440698:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177168.440702:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177168.440706:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177168.888583:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177168.888589:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177168.888591:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177168.888602:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177168.888605:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177168.888608:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177168.888610:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177168.888612:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177168.888614:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177169.464637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177169.464642:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177169.464647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177169.464648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177169.464650:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177169.464651:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177169.464702:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177169.464708:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177169.464712:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177169.656637:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302065357, at 4302065408
00000100:00000040:1.0:1786177169.656649:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302065357, at 4302065408
00000100:00000010:0.0:1786177169.656651:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad371e0.
00000100:00000010:1.0:1786177169.656665:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37b40.
00000100:00000001:0.0:1786177169.912628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177169.912634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177169.912636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177169.912640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177169.912643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177169.912646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177169.912649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177169.912651:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177169.912653:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177170.488576:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177170.488583:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177170.488588:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177170.488590:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177170.488593:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177170.488594:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177170.488644:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177170.488648:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177170.488653:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177170.936622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:1.0:1786177170.936627:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177170.936629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177170.936632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177170.936634:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:0.0:1786177170.936637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786177170.936638:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177170.936643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1786177170.936643:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177170.936648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177170.936649:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786177170.936654:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177170.936658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:1.0:1786177170.936658:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7394
00000100:00000001:0.0:1786177170.936664:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177170.936670:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177170.936676:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177170.936679:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177170.936682:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177170.936685:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177170.936687:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177170.936688:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177170.936692:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177170.936693:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:1.0:1786177170.936696:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177170.936698:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:1.0:1786177170.936700:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:1.0:1786177170.936702:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:1.0:1786177170.936703:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177170.936705:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177170.936706:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177170.936707:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177170.936709:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:1.0:1786177170.936711:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177170.936713:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177170.936716:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177170.936718:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177170.936720:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177170.936722:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177170.936723:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177170.936731:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa5c00.
00000020:00000040:1.0:1786177170.936733:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1786177170.936737:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177170.936738:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177170.936740:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177170.936742:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177170.936745:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177170.936752:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b03800.
00000100:00000001:1.0:1786177170.936757:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177170.936764:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177170.936773:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177170.936776:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177170.936780:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d746f547] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177170.936797:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177170.936800:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177170.936803:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1786177170.936804:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177170.936805:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177170.936807:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177170.936809:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7394
00000400:00000200:1.0:1786177170.936811:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177170.936813:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177170.936815:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177170.936817:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177170.936819:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177170.936822:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177170.936824:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177170.936825:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177170.936826:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:1.0:1786177170.936827:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177170.936829:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:1.0:1786177170.936830:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:1.0:1786177170.936833:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:3.0:1786177170.936833:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177170.936838:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1786177170.936839:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177170.936842:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1786177170.936843:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177170.936847:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177170.936849:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177170.936853:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000040:3.0:1786177170.936853:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1786177170.936860:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177170.936864:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177170.936868:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177170.936872:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177170.936874:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177170.936875:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177170.936878:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177170.936879:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177170.936881:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177170.936882:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:3.0:1786177170.936886:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1786177170.936887:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00100000:3.0:1786177170.936891:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856345472:192.168.202.110@tcp:250:kworker.0
00000008:00000001:1.0:1786177170.936892:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177170.936896:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.936901:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:1.0:1786177170.936902:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa6680.
00000020:00000040:1.0:1786177170.936906:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177170.936906:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177170.936911:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177170.936911:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177170.936914:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177170.936915:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1786177170.936916:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177170.936919:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177170.936920:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177170.936925:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b01800.
02000000:00000010:3.0:1786177170.936925:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177170.936932:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177170.936933:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177170.936942:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177170.936947:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177170.936954:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177170.936955:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177170.936957:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000040:1.0:1786177170.936958:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dced6303] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177170.936967:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177170.936967:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00080000:1.0:1786177170.936971:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177170.936974:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1786177170.936975:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177170.936977:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1786177170.936978:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856345472, portal 25
00000100:00000001:1.0:1786177170.936979:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177170.936983:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00080000:1.0:1786177170.936984:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7394
00000100:00000001:3.0:1786177170.936987:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751749632 : -114547957801984 : ffff97d1b9aa5c00)
00000400:00000200:1.0:1786177170.936992:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177170.936998:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1786177170.936999:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1786177170.937000:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177170.937007:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177170.937011:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177170.937029:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177170.937031:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177170.937036:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:3.0:1786177170.937037:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000001:1.0:1786177170.937039:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177170.937041:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000010:3.0:1786177170.937042:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000020:00000001:1.0:1786177170.937043:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177170.937046:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:1.0:1786177170.937049:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000200:3.0:1786177170.937050:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856345472, offset 0
00000020:00000001:1.0:1786177170.937051:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:1.0:1786177170.937057:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177170.937059:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177170.937060:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177170.937061:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177170.937063:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:1.0:1786177170.937064:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177170.937066:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177170.937067:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177170.937070:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177170.937074:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:1.0:1786177170.937075:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177170.937079:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177170.937081:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1786177170.937083:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177170.937085:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177170.937089:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:1.0:1786177170.937091:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa6300.
00000020:00000040:1.0:1786177170.937095:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:3.0:1786177170.937097:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:1.0:1786177170.937098:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177170.937101:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177170.937103:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1786177170.937103:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:1.0:1786177170.937105:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177170.937107:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177170.937110:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000010:1.0:1786177170.937112:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b03000.
00000400:00000200:3.0:1786177170.937116:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1786177170.937118:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177170.937120:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00100000:1.0:1786177170.937126:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177170.937127:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254870:254870:256:4294967295] 192.168.202.110@tcp LPNI seq info [254859:254859:8:4294967295]
00000400:00000200:3.0:1786177170.937138:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177170.937151:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177170.937151:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1786177170.937156:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177170.937160:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:1.0:1786177170.937163:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a345583] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:3.0:1786177170.937167:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000001:1.0:1786177170.937174:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177170.937178:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:3.0:1786177170.937178:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:1.0:1786177170.937185:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1786177170.937186:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177170.937187:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177170.937189:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:3.0:1786177170.937190:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00080000:1.0:1786177170.937193:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7394
00000800:00000200:3.1:1786177170.937199:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:1.0:1786177170.937202:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:3.1:1786177170.937204:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000400:00000200:1.0:1786177170.937212:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000100:3.0:1786177170.937212:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177170.937216:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00080000:1.0:1786177170.937219:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1786177170.937222:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:1.0:1786177170.937226:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177170.937231:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1786177170.937233:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177170.937237:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000100:00000040:1.0:1786177170.937240:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:3.0:1786177170.937241:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:1.0:1786177170.937247:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177170.937248:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:1.0:1786177170.937253:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000200:3.0:1786177170.937254:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1786177170.937257:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:1.0:1786177170.937262:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177170.937266:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:3.0:1786177170.937268:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000001:1.0:1786177170.937271:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177170.937273:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1786177170.937277:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:3.0:1786177170.937285:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177170.937288:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177170.937288:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177170.937294:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177170.937295:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177170.937300:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177170.937303:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177170.937307:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177170.937307:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302070477
00000020:00000040:1.0:1786177170.937313:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000010:3.0:1786177170.937315:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad372a0.
00000100:00080000:1.0:1786177170.937320:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177170.937324:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1786177170.937325:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177170.937329:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177170.937330:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302070477
02000000:00000001:1.0:1786177170.937333:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:3.0:1786177170.937335:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37a80.
02000000:00000001:1.0:1786177170.937339:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.937340:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1786177170.937346:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177170.937347:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177170.937351:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1786177170.937352:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177170.937353:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000008:00000001:1.0:1786177170.937359:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177170.937359:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177170/real 1786177170]  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177170.937365:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177170.937373:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa5880.
00000400:00000001:3.0:1786177170.937374:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177170.937376:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:1.0:1786177170.937380:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000400:00000010:3.0:1786177170.937380:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177170.937385:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
02000000:00000001:1.0:1786177170.937387:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:3.0:1786177170.937387:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
02000000:00000001:1.0:1786177170.937391:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.937392:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1786177170.937396:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:3.0:1786177170.937397:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786177170.937401:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177170.937405:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177170.937410:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1786177170.937412:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b00c00.
00000100:00000001:3.0:1786177170.937420:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177170.937421:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.937422:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.937426:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177170.937427:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177170.937432:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1786177170.937433:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177170.937459:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177170.937464:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.937471:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177170.937473:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbacf3d] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177170.937481:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177170.937493:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.937500:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177170.937501:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177170.937507:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.937530:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177170.937531:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.937533:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177170.937535:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177170.937537:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177170.937542:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177170.937546:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177170.937552:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1786177170.937560:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1786177170.937565:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177170.937570:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1786177170.937571:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856345472:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177170.937577:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177170.937581:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:1.0:1786177170.937584:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7394
00000100:00000001:3.0:1786177170.937585:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177170.937590:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5c00 x1872934856345472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786177170.937594:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177170.937602:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000400:00000200:1.0:1786177170.937604:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:3.0:1786177170.937606:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00080000:1.0:1786177170.937608:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
02000000:00000010:3.0:1786177170.937609:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177170.937616:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000040:1.0:1786177170.937618:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000020:00000001:3.0:1786177170.937618:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177170.937622:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:1.0:1786177170.937624:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000020:00000001:3.0:1786177170.937625:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177170.937628:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03800.
00000100:00000040:1.0:1786177170.937629:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177170.937633:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000001:1.0:1786177170.937634:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177170.937639:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177170.937640:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:1.0:1786177170.937642:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
02000000:00000001:3.0:1786177170.937643:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000020:00000040:1.0:1786177170.937645:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000010:3.0:1786177170.937646:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5c00.
00000020:00000001:1.0:1786177170.937650:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:3.0:1786177170.937653:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000020:00000001:1.0:1786177170.937656:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:3.0:1786177170.937656:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.937661:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177170.937666:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177170.937670:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177170.937670:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177170.937674:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177170.937675:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177170.937677:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177170.937681:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177170.937681:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177170.937684:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1786177170.937687:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000040:3.0:1786177170.937690:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1786177170.937696:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177170.937702:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177170.937703:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.937704:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177170.937710:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177170.937711:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177170.937715:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177170.937717:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177170.937718:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1786177170.937721:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345600:192.168.202.110@tcp:38:kworker.0
02000000:00000001:1.0:1786177170.937724:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.937725:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1786177170.937729:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:3.0:1786177170.937730:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177170.937734:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1786177170.937735:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000001:3.0:1786177170.937736:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177170.937740:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
00000008:00000001:1.0:1786177170.937741:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177170.937743:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177170.937748:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177170.937749:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177170.937755:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000010:1.0:1786177170.937758:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa5500.
00000400:00000010:3.0:1786177170.937759:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000020:00000040:1.0:1786177170.937764:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000200:3.0:1786177170.937765:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856345600, portal 10
00000100:00000001:3.0:1786177170.937770:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
02000000:00000001:1.0:1786177170.937772:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177170.937773:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751752320 : -114547957799296 : ffff97d1b9aa6680)
02000000:00000001:1.0:1786177170.937777:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177170.937782:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:3.0:1786177170.937782:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786177170.937787:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177170.937792:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.937792:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177170.937797:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177170.937800:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
02000000:00000010:1.0:1786177170.937801:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b02c00.
00000100:00000200:3.0:1786177170.937803:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856345600, offset 0
00000400:00000200:3.0:1786177170.937808:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1786177170.937809:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177170.937813:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177170.937819:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00100000:1.0:1786177170.937820:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177170.937823:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177170.937828:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177170.937833:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177170.937838:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177170.937844:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254871:254871:256:4294967295] 192.168.202.110@tcp LPNI seq info [254860:254860:8:4294967295]
00000100:00000001:1.0:1786177170.937847:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177170.937851:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177170.937854:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177170.937861:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1786177170.937866:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bf052360] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1786177170.937869:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177170.937874:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177170.937883:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177170.937888:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:1.0:1786177170.937889:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177170.937894:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1786177170.937898:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7404)
00000800:00000200:3.1:1786177170.937898:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786177170.937906:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177170.937907:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177170.937910:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177170.937912:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177170.937914:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177170.937916:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177170.937918:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177170.937923:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.937932:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177170.937935:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.937942:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.937943:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177170.937947:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177170.937951:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177170.937955:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177170.937957:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302066698
00000100:00000001:3.0:1786177170.937959:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302070477
00000100:00000001:3.0:1786177170.937962:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302066698
00000100:00000001:3.0:1786177170.937964:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302070477
00000100:00000001:3.0:1786177170.937967:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177170.937969:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.937971:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177170.937975:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177170/real 1786177170]  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177170.937982:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177170.937983:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177170.937984:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177170.937986:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177170.937987:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177170.937988:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177170.937990:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177170.937996:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938000:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177170.938001:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938002:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177170.938003:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177170.938005:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938009:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177170.938012:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938035:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177170.938038:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938039:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177170.938041:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938043:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177170.938045:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177170.938047:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177170.938051:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177170.938053:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177170.938057:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177170.938063:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345600:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177170.938065:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177170.938068:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938072:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177170.938073:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177170.938074:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177170.938076:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177170.938077:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177170.938079:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177170.938081:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177170.938082:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177170.938083:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177170.938084:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177170.938088:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177170.938089:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177170.938091:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177170.938093:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177170.938095:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177170.938098:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856345728, portal 10
00000100:00000001:3.0:1786177170.938099:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177170.938100:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751751424 : -114547957800192 : ffff97d1b9aa6300)
00000100:00000040:3.0:1786177170.938104:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938108:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177170.938110:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177170.938111:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177170.938113:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856345728, offset 0
00000400:00000200:3.0:1786177170.938115:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177170.938118:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177170.938120:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177170.938122:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177170.938124:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177170.938126:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177170.938128:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177170.938130:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254872:254872:256:4294967295] 192.168.202.110@tcp LPNI seq info [254861:254861:8:4294967295]
00000400:00000200:3.0:1786177170.938133:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177170.938137:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177170.938140:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177170.938142:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177170.938144:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177170.938146:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786177170.938149:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177170.938151:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786177170.938153:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177170.938155:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177170.938157:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177170.938159:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177170.938161:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177170.938163:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177170.938164:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177170.938166:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938172:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177170.938173:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938177:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.938179:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177170.938180:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177170.938182:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177170.938185:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177170.938186:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302066823
00000100:00000001:3.0:1786177170.938188:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302070477
00000100:00000001:3.0:1786177170.938189:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302066823
00000100:00000001:3.0:1786177170.938190:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302070477
00000100:00000001:3.0:1786177170.938192:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177170.938193:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.938194:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177170.938196:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177170/real 1786177170]  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177170.938201:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177170.938201:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177170.938203:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177170.938204:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177170.938205:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177170.938207:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177170.938208:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177170.938213:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938218:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177170.938219:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938220:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177170.938221:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177170.938223:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938227:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177170.938230:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938234:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177170.938235:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938236:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177170.938237:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938238:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177170.938239:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177170.938240:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177170.938242:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177170.938244:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177170.938246:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177170.938250:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177170.938253:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177170.938255:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938275:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177170.938276:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177170.938278:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177170.938279:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177170.938281:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177170.938283:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177170.938285:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177170.938286:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177170.938287:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177170.938288:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177170.938291:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177170.938292:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177170.938295:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177170.938297:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177170.938299:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177170.938301:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856345856, portal 4
00000100:00000001:3.0:1786177170.938302:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177170.938303:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751748736 : -114547957802880 : ffff97d1b9aa5880)
00000100:00000040:3.0:1786177170.938307:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938312:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177170.938313:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177170.938315:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177170.938317:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856345856, offset 0
00000400:00000200:3.0:1786177170.938319:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177170.938321:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177170.938323:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177170.938325:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177170.938327:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177170.938329:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177170.938330:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177170.938332:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254873:254873:256:4294967295] 192.168.202.110@tcp LPNI seq info [254862:254862:8:4294967295]
00000400:00000200:3.0:1786177170.938335:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177170.938340:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177170.938343:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177170.938345:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177170.938348:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177170.938350:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786177170.938353:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177170.938355:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786177170.938358:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177170.938359:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177170.938361:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177170.938363:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177170.938365:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177170.938367:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177170.938368:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177170.938370:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938376:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177170.938378:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938382:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.938383:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177170.938385:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177170.938387:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177170.938389:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177170.938390:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302066698
00000100:00000001:3.0:1786177170.938392:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302070477
00000100:00000001:3.0:1786177170.938393:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302066698
00000100:00000001:3.0:1786177170.938395:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302070477
00000100:00000001:3.0:1786177170.938396:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177170.938397:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.938399:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177170.938401:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177170/real 1786177170]  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177170.938407:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177170.938408:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177170.938409:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177170.938410:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177170.938411:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177170.938413:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177170.938415:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177170.938420:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938424:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177170.938425:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938427:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177170.938428:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177170.938430:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938434:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177170.938437:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938443:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177170.938446:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938448:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177170.938449:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938451:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177170.938452:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177170.938455:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177170.938459:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177170.938461:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177170.938466:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177170.938476:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177170.938480:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938482:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177170.938487:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa6680 x1872934856345600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938511:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177170.938512:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177170.938515:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177170.938517:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177170.938519:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177170.938520:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177170.938522:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177170.938524:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b01800.
02000000:00000001:3.0:1786177170.938527:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177170.938528:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177170.938530:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa6680.
00000100:00000001:3.0:1786177170.938532:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177170.938534:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938535:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177170.938538:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa6300 x1872934856345728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938558:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177170.938559:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177170.938560:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177170.938562:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177170.938563:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177170.938564:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177170.938565:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177170.938566:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03000.
02000000:00000001:3.0:1786177170.938568:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177170.938569:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177170.938570:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa6300.
00000100:00000001:3.0:1786177170.938572:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177170.938573:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938574:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177170.938576:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5880 x1872934856345856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938580:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177170.938581:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177170.938582:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177170.938583:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177170.938584:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177170.938584:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177170.938585:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177170.938586:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00c00.
02000000:00000001:3.0:1786177170.938588:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177170.938588:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177170.938589:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5880.
00000100:00000001:3.0:1786177170.938590:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177170.938591:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938592:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938594:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177170.938595:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.938597:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177170.938597:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177170.938598:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177170.938601:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938605:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177170.938605:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177170.938607:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177170.938608:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177170.938609:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177170.938611:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177170.938613:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177170.938615:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177170.938616:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177170.938617:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177170.938619:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177170.938620:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177170.938623:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177170.938624:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177170.938627:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177170.938629:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856345984, portal 4
00000100:00000001:3.0:1786177170.938630:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177170.938631:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751747840 : -114547957803776 : ffff97d1b9aa5500)
00000100:00000040:3.0:1786177170.938635:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938640:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177170.938641:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177170.938643:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177170.938644:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856345984, offset 0
00000400:00000200:3.0:1786177170.938647:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177170.938649:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177170.938652:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177170.938654:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177170.938655:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177170.938657:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177170.938659:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177170.938661:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254874:254874:256:4294967295] 192.168.202.110@tcp LPNI seq info [254863:254863:8:4294967295]
00000400:00000200:3.0:1786177170.938664:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177170.938668:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177170.938670:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177170.938672:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177170.938675:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177170.938676:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786177170.938679:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177170.938681:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786177170.938683:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177170.938685:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177170.938687:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177170.938689:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177170.938691:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177170.938692:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177170.938694:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177170.938696:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938701:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177170.938703:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938707:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.938708:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177170.938710:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177170.938712:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177170.938714:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177170.938716:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302066707
00000100:00000001:3.0:1786177170.938717:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302070477
00000100:00000001:3.0:1786177170.938719:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302066707
00000100:00000001:3.0:1786177170.938720:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302070477
00000100:00000001:3.0:1786177170.938722:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177170.938723:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.938724:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177170.938726:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177170/real 1786177170]  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177170.938731:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177170.938731:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177170.938732:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177170.938734:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177170.938735:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177170.938737:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177170.938739:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177170.938744:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938748:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177170.938749:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938750:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177170.938751:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177170.938753:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938757:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177170.938759:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938763:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177170.938765:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938766:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177170.938767:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938768:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177170.938769:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177170.938771:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177170.938772:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177170.938774:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177170.938776:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177170.938781:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856345984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177170.938783:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938784:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177170.938786:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5500 x1872934856345984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177170.938790:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177170.938791:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177170.938792:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177170.938794:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177170.938794:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177170.938795:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177170.938796:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177170.938797:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b02c00.
02000000:00000001:3.0:1786177170.938799:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177170.938799:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177170.938800:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5500.
00000100:00000001:3.0:1786177170.938802:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177170.938802:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938803:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177170.938805:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177170.938806:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177170.938807:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177170.938808:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177171.512664:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177171.512673:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177171.512676:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177171.512680:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177171.512685:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177171.512688:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177171.512690:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177171.512693:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177171.512697:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177171.512699:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177171.512700:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177171.512702:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177171.512704:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177171.512706:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177171.512708:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177171.512710:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177171.512713:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177171.512715:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177171.512813:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177171.512816:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177171.512819:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177171.960632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177171.960640:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177171.960643:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177171.960648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177171.960652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177171.960656:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177171.960659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177171.960662:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177171.960665:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177172.536610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177172.536616:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177172.536620:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177172.536621:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177172.536623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177172.536624:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177172.536673:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177172.536676:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177172.536679:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177172.984630:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177172.984639:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177172.984642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177172.984648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177172.984652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177172.984657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177172.984660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177172.984663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177172.984666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177173.560699:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177173.560709:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177173.560717:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177173.560719:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177173.560723:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177173.560728:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177173.560731:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177173.560735:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177173.560739:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177173.560742:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177173.560745:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177173.560747:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177173.560826:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177173.560830:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177173.560833:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177174.008583:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177174.008593:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177174.008595:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177174.008600:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177174.008604:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177174.008607:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177174.008610:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177174.008613:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177174.008615:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177174.584642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177174.584650:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177174.584656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177174.584658:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177174.584662:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177174.584663:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177174.584760:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177174.584766:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177174.584770:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177174.776614:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302070477, at 4302070528
00000100:00000040:1.0:1786177174.776614:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302070477, at 4302070528
00000100:00000010:1.0:1786177174.776628:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37a80.
00000100:00000010:0.0:1786177174.776630:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad372a0.
00000100:00000001:0.0:1786177175.032612:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177175.032620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177175.032621:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177175.032625:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177175.032628:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177175.032631:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177175.032634:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177175.032635:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177175.032638:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177175.608597:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177175.608603:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177175.608607:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177175.608608:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177175.608611:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177175.608612:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177175.608701:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177175.608709:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177175.608714:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786177176.056689:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177176.056692:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177176.056700:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786177176.056701:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177176.056705:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177176.056706:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177176.056712:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1786177176.056712:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177176.056725:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177176.056725:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177176.056734:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177176.056741:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7399
00000100:00000001:3.0:1786177176.056742:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000400:00000200:0.0:1786177176.056753:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177176.056756:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.056762:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177176.056772:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177176.056776:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177176.056779:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.056783:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177176.056785:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.056787:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177176.056791:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177176.056794:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177176.056797:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177176.056800:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177176.056802:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177176.056804:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177176.056806:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.056807:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177176.056809:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.056810:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177176.056812:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177176.056814:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177176.056816:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177176.056819:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177176.056823:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177176.056826:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177176.056827:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177176.056829:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177176.056839:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41ad80.
00000020:00000040:0.0:1786177176.056842:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177176.056846:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177176.056847:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177176.056851:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177176.056853:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177176.056858:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177176.056866:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4000.
00000100:00000001:0.0:1786177176.056875:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177176.056889:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177176.056902:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177176.056907:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177176.056913:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034e2c676] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177176.056942:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177176.056945:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177176.056948:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177176.056949:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177176.056950:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177176.056953:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177176.056955:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7399
00000400:00000200:0.0:1786177176.056959:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177176.056962:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177176.056964:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177176.056983:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.056986:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177176.056987:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1786177176.056989:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.056994:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177176.056995:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1786177176.056997:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177176.057000:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:3.0:1786177176.057001:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1786177176.057002:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177176.057009:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177176.057013:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177176.057018:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177176.057020:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177176.057025:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.057034:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177176.057042:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.057052:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177176.057052:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177176.057058:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177176.057060:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177176.057067:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1786177176.057067:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1786177176.057072:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177176.057073:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177176.057084:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177176.057084:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177176.057092:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856346112:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1786177176.057095:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177176.057097:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177176.057098:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177176.057102:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:3.0:1786177176.057102:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177176.057109:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1786177176.057110:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1786177176.057112:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:0.0:1786177176.057114:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177176.057117:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177176.057122:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41aa00.
02000000:00000010:3.0:1786177176.057123:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
00000020:00000040:0.0:1786177176.057126:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177176.057127:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177176.057138:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177176.057139:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177176.057141:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177176.057142:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177176.057144:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177176.057152:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5800.
00000400:00000200:3.0:1786177176.057152:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177176.057162:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177176.057163:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177176.057185:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00100000:0.0:1786177176.057186:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1786177176.057199:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856346112, portal 25
00000100:00000001:0.0:1786177176.057200:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177176.057208:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.057209:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177176.057218:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761666432 : -114547947885184 : ffff97d1ba41ad80)
00000100:00000040:0.0:1786177176.057223:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da0255a4] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177176.057233:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177176.057240:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177176.057252:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177176.057253:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:0.0:1786177176.057266:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000040:3.0:1786177176.057267:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177176.057277:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:3.0:1786177176.057277:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00080000:0.0:1786177176.057283:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1786177176.057286:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856346112, offset 0
00000100:00000001:0.0:1786177176.057287:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177176.057299:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7399
00000400:00000200:3.0:1786177176.057303:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177176.057313:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177176.057319:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177176.057320:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177176.057328:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177176.057331:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.057338:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177176.057339:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:0.0:1786177176.057349:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177176.057352:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1786177176.057358:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177176.057365:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:3.0:1786177176.057365:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:0.0:1786177176.057369:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000200:3.0:1786177176.057373:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:0.0:1786177176.057374:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177176.057382:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:3.0:1786177176.057383:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000001:0.0:1786177176.057390:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177176.057397:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254875:254875:256:4294967295] 192.168.202.110@tcp LPNI seq info [254864:254864:8:4294967295]
00000020:00000001:0.0:1786177176.057398:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177176.057414:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177176.057415:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177176.057422:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177176.057425:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.057428:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177176.057430:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786177176.057439:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177176.057446:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177176.057451:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:0.0:1786177176.057452:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177176.057460:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177176.057466:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177176.057467:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177176.057471:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177176.057474:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000800:00000010:3.0:1786177176.057474:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000008:00000001:0.0:1786177176.057483:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177176.057489:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177176.057490:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:0.0:1786177176.057495:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419500.
00000800:00000200:3.0:1786177176.057496:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000040:0.0:1786177176.057500:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177176.057506:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177176.057508:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177176.057509:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177176.057535:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177176.057536:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177176.057538:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177176.057542:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
02000000:00000010:0.0:1786177176.057544:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7800.
00000100:00000001:0.0:1786177176.057555:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1786177176.057560:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00100000:0.0:1786177176.057561:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:3.0:1786177176.057564:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177176.057578:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1786177176.057580:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177176.057587:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177176.057593:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:0.0:1786177176.057598:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b6718a9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177176.057601:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177176.057608:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177176.057615:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1786177176.057621:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177176.057623:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177176.057632:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177176.057639:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1786177176.057641:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000040:3.0:1786177176.057645:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177176.057646:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177176.057653:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177176.057662:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177176.057665:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177176.057666:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:0.0:1786177176.057673:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7399
00000100:00000001:3.0:1786177176.057675:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177176.057682:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177176.057685:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177176.057688:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302075598
00000400:00000200:0.0:1786177176.057692:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177176.057696:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000010:3.0:1786177176.057697:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37d80.
00000100:00000040:0.0:1786177176.057707:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.057712:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177176.057713:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:0.0:1786177176.057723:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177176.057725:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302075598
00000100:00000001:0.0:1786177176.057726:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177176.057736:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000010:3.0:1786177176.057737:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37e40.
00000020:00000040:0.0:1786177176.057738:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177176.057744:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177176.057747:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000020:00000040:0.0:1786177176.057748:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:3.0:1786177176.057753:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1786177176.057754:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177176.057770:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:3.0:1786177176.057770:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.057783:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:0.0:1786177176.057786:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00100000:3.0:1786177176.057791:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177176/real 1786177176]  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177176.057792:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177176.057800:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.057808:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177176.057816:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177176.057829:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177176.057839:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177176.057851:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177176.057857:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000001:3.0:1786177176.057858:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:0.0:1786177176.057864:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177176.057864:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1786177176.057872:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000010:3.0:1786177176.057872:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177176.057881:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000008:00100000:0.0:1786177176.057883:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177176.057890:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177176.057892:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177176.057892:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177176.057898:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000010:0.0:1786177176.057902:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a680.
00000100:00000200:3.0:1786177176.057903:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177176.057910:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177176.057920:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177176.057925:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177176.057925:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177176.057934:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177176.057939:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177176.057945:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.057950:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000010:0.0:1786177176.057955:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000001:3.0:1786177176.057955:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.057980:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177176.057983:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177176.057984:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177176.057990:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177176.057992:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177176.058056:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177176.058057:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177176.058064:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177176.058070:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177176.058073:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000911ee1e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058128:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177176.058133:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.058135:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177176.058139:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177176.058139:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177176.058147:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177176.058150:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177176.058151:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177176.058153:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177176.058155:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177176.058157:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177176.058163:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177176.058164:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177176.058173:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7399
00000100:00080000:3.0:1786177176.058173:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177176.058181:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000400:00000200:0.0:1786177176.058182:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177176.058193:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1786177176.058194:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177176.058201:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177176.058213:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.058224:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177176.058231:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177176.058239:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00100000:3.0:1786177176.058239:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856346112:192.168.202.110@tcp:250:kworker.0
00000020:00000001:0.0:1786177176.058249:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177176.058252:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:3.0:1786177176.058252:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1786177176.058259:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177176.058260:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1786177176.058265:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000040:3.0:1786177176.058267:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41ad80 x1872934856346112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177176.058270:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:0.0:1786177176.058279:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786177176.058292:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177176.058296:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000001:0.0:1786177176.058297:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1786177176.058301:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000040:0.0:1786177176.058303:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000010:3.0:1786177176.058305:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
00000100:00000001:0.0:1786177176.058308:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177176.058311:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000040:0.0:1786177176.058312:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000020:00000001:3.0:1786177176.058319:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00080000:0.0:1786177176.058320:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000020:00000040:3.0:1786177176.058322:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:0.0:1786177176.058324:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177176.058330:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000020:00000001:3.0:1786177176.058330:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177176.058339:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177176.058340:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177176.058348:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177176.058349:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000001:0.0:1786177176.058359:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:3.0:1786177176.058359:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000008:00100000:0.0:1786177176.058367:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000010:3.0:1786177176.058367:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41ad80.
00000008:00000001:0.0:1786177176.058376:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177176.058379:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.058380:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177176.058385:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:0.0:1786177176.058390:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a300.
00000100:00000001:3.0:1786177176.058391:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1786177176.058402:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177176.058405:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177176.058406:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177176.058407:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177176.058407:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:0.0:1786177176.058414:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177176.058415:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.058424:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:0.0:1786177176.058425:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.058429:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
02000000:00000010:0.0:1786177176.058431:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000100:00000001:3.0:1786177176.058431:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1786177176.058439:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177176.058443:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177176.058449:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177176.058481:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177176.058487:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177176.058487:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177176.058495:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177176.058495:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177176.058502:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1786177176.058508:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b0d3569] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177176.058508:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177176.058536:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856346240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177176.058555:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177176.058559:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1786177176.058561:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177176.058561:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177176.058578:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:0.0:1786177176.058583:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7409)
02000000:00000010:3.0:1786177176.058584:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177176.058599:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177176.058605:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177176.058608:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177176.058610:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177176.058613:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856346240, portal 10
00000100:00000001:3.0:1786177176.058615:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177176.058616:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761665536 : -114547947886080 : ffff97d1ba41aa00)
00000100:00000040:3.0:1786177176.058622:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058628:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177176.058629:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177176.058631:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177176.058632:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856346240, offset 0
00000400:00000200:3.0:1786177176.058635:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177176.058637:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177176.058641:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177176.058643:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177176.058645:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177176.058646:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177176.058648:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177176.058650:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254876:254876:256:4294967295] 192.168.202.110@tcp LPNI seq info [254865:254865:8:4294967295]
00000400:00000200:3.0:1786177176.058653:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177176.058658:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177176.058661:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177176.058663:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177176.058665:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177176.058668:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786177176.058671:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177176.058673:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1786177176.058675:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177176.058677:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177176.058679:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177176.058681:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177176.058684:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177176.058685:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177176.058687:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177176.058689:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058694:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177176.058696:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058700:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.058702:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177176.058704:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177176.058707:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177176.058709:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177176.058711:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302071819
00000100:00000001:3.0:1786177176.058713:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302075598
00000100:00000001:3.0:1786177176.058714:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302071819
00000100:00000001:3.0:1786177176.058716:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302075598
00000100:00000001:3.0:1786177176.058717:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177176.058718:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.058719:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177176.058722:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177176/real 1786177176]  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177176.058726:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177176.058727:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177176.058729:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177176.058730:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177176.058731:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177176.058733:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177176.058735:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177176.058740:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058744:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177176.058744:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.058745:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177176.058746:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177176.058748:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058752:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177176.058755:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058759:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177176.058761:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.058762:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177176.058763:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.058764:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177176.058766:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177176.058768:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177176.058770:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177176.058772:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177176.058774:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177176.058779:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856346240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177176.058782:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177176.058784:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058789:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177176.058790:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177176.058791:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177176.058793:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177176.058794:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177176.058795:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856346368:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177176.058797:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177176.058799:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177176.058800:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177176.058801:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177176.058803:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177176.058805:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177176.058807:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177176.058808:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177176.058810:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177176.058812:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856346368, portal 10
00000100:00000001:3.0:1786177176.058813:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177176.058814:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761660160 : -114547947891456 : ffff97d1ba419500)
00000100:00000040:3.0:1786177176.058818:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058822:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177176.058823:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177176.058824:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177176.058826:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856346368, offset 0
00000400:00000200:3.0:1786177176.058828:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177176.058830:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177176.058832:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177176.058834:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177176.058836:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177176.058837:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177176.058839:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177176.058841:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254877:254877:256:4294967295] 192.168.202.110@tcp LPNI seq info [254866:254866:8:4294967295]
00000400:00000200:3.0:1786177176.058844:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177176.058847:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177176.058850:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177176.058851:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177176.058854:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177176.058856:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786177176.058858:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177176.058860:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1786177176.058863:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177176.058864:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177176.058866:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177176.058868:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177176.058870:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177176.058871:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177176.058873:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177176.058875:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058880:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177176.058882:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058886:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.058887:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177176.058889:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177176.058891:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177176.058893:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177176.058894:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302071944
00000100:00000001:3.0:1786177176.058895:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302075598
00000100:00000001:3.0:1786177176.058897:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302071944
00000100:00000001:3.0:1786177176.058899:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302075598
00000100:00000001:3.0:1786177176.058900:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177176.058901:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.058903:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177176.058905:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177176/real 1786177176]  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177176.058909:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177176.058910:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177176.058911:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177176.058912:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177176.058913:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177176.058914:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177176.058916:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177176.058921:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058926:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177176.058927:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.058927:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177176.058928:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177176.058930:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058934:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177176.058936:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058941:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177176.058942:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.058943:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177176.058944:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.058945:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177176.058947:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177176.058948:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177176.058951:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177176.058952:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177176.058954:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177176.058959:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856346368:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177176.058961:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177176.058963:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.058981:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177176.058982:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177176.058983:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177176.058984:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177176.058986:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177176.058987:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856346496:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177176.058990:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177176.058991:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177176.058992:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177176.058993:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177176.058996:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177176.058997:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177176.058999:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177176.059001:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177176.059003:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177176.059005:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856346496, portal 4
00000100:00000001:3.0:1786177176.059006:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177176.059007:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761664640 : -114547947886976 : ffff97d1ba41a680)
00000100:00000040:3.0:1786177176.059011:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059016:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177176.059017:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177176.059019:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177176.059020:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856346496, offset 0
00000400:00000200:3.0:1786177176.059022:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177176.059025:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177176.059027:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177176.059028:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177176.059030:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177176.059032:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177176.059033:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177176.059035:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254878:254878:256:4294967295] 192.168.202.110@tcp LPNI seq info [254867:254867:8:4294967295]
00000400:00000200:3.0:1786177176.059038:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177176.059042:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177176.059044:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177176.059046:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177176.059048:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177176.059050:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786177176.059053:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177176.059055:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1786177176.059058:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177176.059059:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177176.059061:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177176.059063:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177176.059065:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177176.059066:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177176.059068:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177176.059070:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059075:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177176.059077:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059086:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.059087:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177176.059089:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177176.059091:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177176.059093:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177176.059094:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302071819
00000100:00000001:3.0:1786177176.059096:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302075598
00000100:00000001:3.0:1786177176.059097:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302071819
00000100:00000001:3.0:1786177176.059099:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302075598
00000100:00000001:3.0:1786177176.059100:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177176.059101:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.059102:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177176.059104:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177176/real 1786177176]  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177176.059109:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177176.059109:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177176.059110:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177176.059112:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177176.059113:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177176.059114:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177176.059116:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177176.059121:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059125:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177176.059126:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059127:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177176.059128:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177176.059130:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059134:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177176.059137:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059141:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177176.059143:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059144:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177176.059144:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059145:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177176.059146:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177176.059148:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177176.059150:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177176.059151:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177176.059153:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177176.059158:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856346496:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177176.059160:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059161:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177176.059163:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41aa00 x1872934856346240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059167:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177176.059168:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177176.059170:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177176.059171:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177176.059172:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177176.059173:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177176.059175:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177176.059176:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5800.
02000000:00000001:3.0:1786177176.059178:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177176.059180:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177176.059181:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41aa00.
00000100:00000001:3.0:1786177176.059182:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177176.059183:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059184:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177176.059187:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419500 x1872934856346368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059191:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177176.059192:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177176.059193:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177176.059194:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177176.059194:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177176.059195:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177176.059196:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177176.059197:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7800.
02000000:00000001:3.0:1786177176.059198:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177176.059199:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177176.059200:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419500.
00000100:00000001:3.0:1786177176.059201:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177176.059202:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059203:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177176.059204:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a680 x1872934856346496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059208:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177176.059209:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177176.059209:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177176.059211:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177176.059212:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177176.059212:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177176.059213:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177176.059214:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:3.0:1786177176.059215:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177176.059216:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177176.059217:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a680.
00000100:00000001:3.0:1786177176.059218:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177176.059219:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059221:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059223:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177176.059225:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.059226:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177176.059228:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177176.059229:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177176.059231:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059235:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177176.059236:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177176.059237:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177176.059238:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177176.059240:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177176.059241:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856346624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177176.059243:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177176.059244:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177176.059245:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177176.059246:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177176.059248:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177176.059250:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177176.059252:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177176.059253:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177176.059255:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177176.059257:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856346624, portal 4
00000100:00000001:3.0:1786177176.059258:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177176.059259:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761663744 : -114547947887872 : ffff97d1ba41a300)
00000100:00000040:3.0:1786177176.059263:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059267:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177176.059269:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177176.059270:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177176.059272:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856346624, offset 0
00000400:00000200:3.0:1786177176.059274:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177176.059276:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177176.059278:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177176.059280:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177176.059282:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177176.059284:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177176.059285:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177176.059287:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254879:254879:256:4294967295] 192.168.202.110@tcp LPNI seq info [254868:254868:8:4294967295]
00000400:00000200:3.0:1786177176.059290:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177176.059294:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177176.059296:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177176.059298:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177176.059300:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177176.059302:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786177176.059305:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177176.059306:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1786177176.059309:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177176.059310:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177176.059312:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177176.059314:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177176.059316:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177176.059318:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177176.059319:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177176.059321:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059326:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177176.059328:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059332:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.059333:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177176.059334:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177176.059336:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177176.059338:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177176.059339:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302071827
00000100:00000001:3.0:1786177176.059341:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302075598
00000100:00000001:3.0:1786177176.059342:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302071827
00000100:00000001:3.0:1786177176.059344:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302075598
00000100:00000001:3.0:1786177176.059345:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177176.059346:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.059347:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177176.059349:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177176/real 1786177176]  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177176.059354:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177176.059355:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177176.059356:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177176.059357:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177176.059358:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177176.059359:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177176.059362:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177176.059367:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059370:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177176.059371:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059372:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177176.059373:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177176.059375:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059379:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177176.059381:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059385:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177176.059387:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059388:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177176.059389:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059390:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177176.059391:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177176.059393:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177176.059395:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177176.059396:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177176.059399:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177176.059403:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856346624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177176.059405:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059406:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177176.059408:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a300 x1872934856346624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177176.059412:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177176.059413:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177176.059414:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177176.059415:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177176.059416:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177176.059416:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177176.059417:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177176.059418:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:3.0:1786177176.059420:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177176.059421:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177176.059422:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a300.
00000100:00000001:3.0:1786177176.059423:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177176.059424:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059425:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177176.059427:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177176.059428:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177176.059429:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177176.059431:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177176.632626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177176.632631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177176.632635:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177176.632636:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177176.632638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177176.632640:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177176.632704:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177176.632708:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177176.632711:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177177.080643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177177.080652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177177.080654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177177.080662:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177177.080665:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177177.080668:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177177.080671:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177177.080673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177177.080676:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177177.656642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177177.656649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177177.656655:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177177.656656:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177177.656659:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177177.656661:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177177.656747:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177177.656752:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177177.656756:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177178.104639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177178.104647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177178.104650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177178.104656:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177178.104660:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177178.104665:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177178.104668:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177178.104671:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177178.104674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177178.680629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177178.680635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177178.680638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177178.680639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177178.680642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177178.680643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177178.680712:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177178.680717:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177178.680722:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177179.128629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177179.128638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177179.128640:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177179.128646:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177179.128649:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177179.128653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177179.128657:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177179.128659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177179.128663:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177179.704721:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177179.704728:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177179.704733:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177179.704734:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177179.704737:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177179.704738:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177179.704801:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177179.704805:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177179.704808:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177179.896594:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302075598, at 4302075648
00000100:00000040:1.0:1786177179.896595:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302075598, at 4302075648
00000100:00000010:1.0:1786177179.896613:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37e40.
00000100:00000010:0.0:1786177179.896615:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37d80.
00000100:00000001:3.0:1786177180.152622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177180.152628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177180.152629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177180.152633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177180.152636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177180.152639:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177180.152641:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177180.152644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177180.152646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177180.728650:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177180.728659:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177180.728662:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177180.728665:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177180.728670:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177180.728673:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177180.728675:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177180.728678:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177180.728682:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177180.728683:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177180.728685:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177180.728686:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177180.728688:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177180.728689:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177180.728691:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177180.728693:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177180.728697:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177180.728699:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177180.728762:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177180.728767:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177180.728771:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177181.176653:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1786177181.176657:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177181.176661:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177181.176666:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177181.176671:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:3.0:1786177181.176674:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177181.176676:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177181.176682:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1786177181.176683:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177181.176686:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177181.176690:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177181.176691:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177181.176693:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.176695:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177181.176707:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7404
00000400:00000200:2.0:1786177181.176718:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177181.176725:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177181.176728:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177181.176731:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177181.176735:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177181.176737:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177181.176739:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177181.176742:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177181.176744:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177181.176747:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177181.176750:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177181.176752:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177181.176754:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177181.176756:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177181.176758:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177181.176759:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177181.176761:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177181.176763:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177181.176765:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177181.176767:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177181.176770:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177181.176773:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177181.176775:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177181.176777:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177181.176778:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177181.176788:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786177181.176791:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177181.176796:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177181.176799:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177181.176802:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177181.176806:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177181.176812:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177181.176824:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786177181.176834:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177181.176848:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177181.176862:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177181.176866:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177181.176871:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177181.176890:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177181.176893:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177181.176896:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177181.176897:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177181.176899:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177181.176901:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177181.176904:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7404
00000400:00000200:2.0:1786177181.176908:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177181.176911:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177181.176913:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177181.176931:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177181.176932:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177181.176939:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177181.176941:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1786177181.176945:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177181.176947:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786177181.176952:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177181.176955:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177181.176956:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177181.176958:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177181.176963:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000040:3.0:1786177181.176965:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177181.176967:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177181.176979:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:3.0:1786177181.176993:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1786177181.176998:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177181.176999:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177181.177009:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1786177181.177009:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177181.177016:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1786177181.177021:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177181.177022:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177181.177032:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00100000:3.0:1786177181.177032:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856346752:192.168.202.110@tcp:250:kworker.0
00000020:00000040:2.0:1786177181.177047:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:3.0:1786177181.177048:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177181.177060:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:2.0:1786177181.177061:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177181.177065:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177181.177066:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177181.177079:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177181.177080:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1786177181.177095:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:3.0:1786177181.177095:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:2.0:1786177181.177110:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177181.177110:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177181.177129:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:3.0:1786177181.177134:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:2.0:1786177181.177135:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177181.177144:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177181.177145:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000001:2.0:1786177181.177146:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177181.177154:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786177181.177157:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000400:00000010:3.0:1786177181.177157:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
02000000:00000001:2.0:1786177181.177169:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177181.177171:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177181.177172:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856346752, portal 25
00000100:00000001:2.0:1786177181.177173:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177181.177180:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177181.177180:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177181.177190:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
02000000:00000001:2.0:1786177181.177192:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177181.177200:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1786177181.177201:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786177181.177211:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.177219:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:2.0:1786177181.177221:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1786177181.177228:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177181.177234:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177181.177240:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856346752, offset 0
00000100:00000001:2.0:1786177181.177249:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177181.177255:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177181.177258:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177181.177266:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177181.177276:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177181.177292:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177181.177297:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1786177181.177298:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:2.0:1786177181.177314:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:3.0:1786177181.177317:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786177181.177319:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177181.177322:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1786177181.177324:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1786177181.177334:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177181.177337:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:2.0:1786177181.177340:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7404
00000400:00000200:3.0:1786177181.177346:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177181.177353:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254880:254880:256:4294967295] 192.168.202.110@tcp LPNI seq info [254869:254869:8:4294967295]
00000400:00000200:3.0:1786177181.177360:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177181.177366:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177181.177371:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177181.177376:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177181.177387:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177181.177393:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177181.177394:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1786177181.177399:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177181.177402:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177181.177404:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177181.177405:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177181.177409:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177181.177412:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177181.177414:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:3.0:1786177181.177420:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000001:2.0:1786177181.177422:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000010:3.0:1786177181.177426:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000040:2.0:1786177181.177435:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:3.0:1786177181.177438:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177181.177447:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1786177181.177449:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1786177181.177456:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:3.1:1786177181.177459:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177181.177468:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.1:1786177181.177469:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000020:00000040:2.0:1786177181.177481:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786177181.177490:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000100:3.0:1786177181.177492:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177181.177495:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1786177181.177497:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
02000000:00000001:2.0:1786177181.177503:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177181.177508:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177181.177537:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1786177181.177545:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177181.177546:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1786177181.177556:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:3.0:1786177181.177557:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000008:00100000:2.0:1786177181.177563:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000010:3.0:1786177181.177563:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177181.177572:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000008:00000001:2.0:1786177181.177573:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177181.177582:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177181.177582:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786177181.177591:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786177181.177600:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000001:3.0:1786177181.177604:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1786177181.177605:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177181.177614:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177181.177619:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:3.0:1786177181.177620:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177181.177626:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177181.177633:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.177637:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.177639:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:2.0:1786177181.177640:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786177181.177647:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.177649:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177181.177654:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177181.177655:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177181.177663:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302080718
00000100:00000001:2.0:1786177181.177674:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000010:3.0:1786177181.177676:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37ea0.
00000100:00000001:2.0:1786177181.177677:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177181.177690:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.177691:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177181.177705:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.177706:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302080718
00000100:00080000:2.0:1786177181.177711:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:3.0:1786177181.177712:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37060.
00000100:00080000:2.0:1786177181.177724:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177181.177726:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786177181.177727:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177181.177733:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177181.177738:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177181.177745:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:2.0:1786177181.177746:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7404
00000100:00000001:3.0:1786177181.177761:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177181.177768:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177181.177774:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000400:00000200:2.0:1786177181.177778:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00100000:3.0:1786177181.177782:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177181/real 1786177181]  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786177181.177787:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177181.177801:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177181.177813:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:3.0:1786177181.177814:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177181.177819:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177181.177822:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00000040:2.0:1786177181.177823:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177181.177828:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000100:00000001:2.0:1786177181.177833:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000010:3.0:1786177181.177833:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177181.177840:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:2.0:1786177181.177843:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000200:3.0:1786177181.177846:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177181.177848:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177181.177855:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177181.177862:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177181.177870:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000200:3.0:1786177181.177870:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177181.177880:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786177181.177896:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177181.177905:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177181.177907:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177181.177911:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177181.177912:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177181.177931:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177181.177934:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:2.0:1786177181.177935:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:3.0:1786177181.177938:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177181.177944:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177181.177953:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177181.177954:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177181.177964:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177181.177972:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177181.177978:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786177181.177982:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177181.177987:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177181.177989:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177181.177996:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177181.177997:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177181.178005:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:2.0:1786177181.178007:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.178010:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177181.178018:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.178019:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.178027:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177181.178029:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000010:2.0:1786177181.178031:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000100:00080000:3.0:1786177181.178034:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000020:00000040:2.0:1786177181.178036:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00080000:3.0:1786177181.178042:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
02000000:00000001:2.0:1786177181.178043:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177181.178045:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:2.0:1786177181.178046:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177181.178060:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177181.178066:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:3.0:1786177181.178066:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177181.178072:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177181.178088:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00100000:3.0:1786177181.178089:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856346752:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177181.178104:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.178105:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.178114:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177181.178120:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856346752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177181.178124:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177181.178126:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177181.178130:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177181.178136:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
00000100:00000001:2.0:1786177181.178140:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1786177181.178143:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:2.0:1786177181.178144:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177181.178148:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177181.178150:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000040:2.0:1786177181.178154:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177181.178155:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177181.178162:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786177181.178173:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177181.178174:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00080000:2.0:1786177181.178176:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:3.0:1786177181.178178:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00080000:2.0:1786177181.178182:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177181.178185:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000010:3.0:1786177181.178187:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00080000:2.0:1786177181.178191:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177181.178196:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177181.178197:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177181.178201:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177181.178206:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7404
00000100:00000001:3.0:1786177181.178208:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1786177181.178215:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177181.178220:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000400:00000200:2.0:1786177181.178222:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177181.178223:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177181.178231:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177181.178231:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177181.178236:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177181.178238:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177181.178242:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:3.0:1786177181.178248:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177181.178249:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177181.178260:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177181.178272:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177181.178281:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177181.178287:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177181.178292:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177181.178293:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177181.178298:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786177181.178299:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
02000000:00000001:3.0:1786177181.178303:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:2.0:1786177181.178304:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786177181.178310:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
02000000:00000001:3.0:1786177181.178310:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177181.178321:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177181.178325:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177181.178332:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:3.0:1786177181.178333:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856346880:192.168.202.110@tcp:38:kworker.0
00000100:00000040:2.0:1786177181.178344:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177181.178346:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1786177181.178347:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177181.178351:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
02000000:00000001:3.0:1786177181.178353:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:2.0:1786177181.178354:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177181.178360:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177181.178361:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177181.178363:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177181.178369:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1786177181.178370:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177181.178376:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177181.178378:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000010:3.0:1786177181.178380:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
00000008:00100000:2.0:1786177181.178384:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000001:3.0:1786177181.178388:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1786177181.178389:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177181.178396:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177181.178402:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000400:00000200:3.0:1786177181.178402:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177181.178418:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000020:00000040:2.0:1786177181.178419:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000400:00000010:3.0:1786177181.178431:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
02000000:00000001:2.0:1786177181.178432:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177181.178436:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177181.178436:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856346880, portal 10
00000100:00000001:2.0:1786177181.178448:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177181.178448:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177181.178451:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
02000000:00000001:2.0:1786177181.178453:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177181.178459:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177181.178460:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178468:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000010:2.0:1786177181.178470:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000040:3.0:1786177181.178475:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177181.178479:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177181.178480:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177181.178490:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856346880, offset 0
00000100:00100000:2.0:1786177181.178495:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177181.178506:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177181.178539:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177181.178542:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177181.178546:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177181.178551:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:2.0:1786177181.178553:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177181.178556:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177181.178570:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177181.178582:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177181.178591:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1786177181.178593:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177181.178603:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254881:254881:256:4294967295] 192.168.202.110@tcp LPNI seq info [254870:254870:8:4294967295]
00000100:00000040:2.0:1786177181.178606:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7414)
00000400:00000200:3.0:1786177181.178609:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177181.178625:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177181.178630:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177181.178634:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177181.178637:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177181.178640:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786177181.178643:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177181.178645:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786177181.178648:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177181.178649:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177181.178652:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177181.178654:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177181.178656:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177181.178658:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177181.178660:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177181.178663:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178669:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177181.178671:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178675:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.178676:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177181.178678:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177181.178682:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177181.178684:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177181.178685:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302076939
00000100:00000001:3.0:1786177181.178688:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302080718
00000100:00000001:3.0:1786177181.178689:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302076939
00000100:00000001:3.0:1786177181.178691:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302080718
00000100:00000001:3.0:1786177181.178693:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177181.178694:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.178695:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177181.178698:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177181/real 1786177181]  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177181.178702:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177181.178703:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177181.178704:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177181.178706:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177181.178707:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177181.178708:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177181.178710:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177181.178716:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178720:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177181.178721:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.178722:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177181.178723:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177181.178725:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178729:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177181.178731:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178736:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177181.178738:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.178739:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177181.178740:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.178742:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177181.178744:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177181.178746:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177181.178748:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177181.178750:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177181.178752:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177181.178757:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856346880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177181.178759:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177181.178762:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178767:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177181.178768:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177181.178769:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177181.178771:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177181.178772:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177181.178774:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177181.178776:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177181.178777:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177181.178779:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177181.178780:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177181.178783:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177181.178784:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177181.178786:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177181.178788:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177181.178790:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177181.178792:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856347008, portal 10
00000100:00000001:3.0:1786177181.178793:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177181.178795:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:3.0:1786177181.178798:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178802:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177181.178804:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177181.178805:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177181.178807:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856347008, offset 0
00000400:00000200:3.0:1786177181.178809:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177181.178812:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177181.178813:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177181.178815:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177181.178817:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177181.178819:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177181.178820:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177181.178823:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254882:254882:256:4294967295] 192.168.202.110@tcp LPNI seq info [254871:254871:8:4294967295]
00000400:00000200:3.0:1786177181.178825:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177181.178829:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177181.178832:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177181.178834:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177181.178836:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177181.178838:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786177181.178841:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177181.178842:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786177181.178845:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177181.178846:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177181.178848:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177181.178850:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177181.178852:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177181.178853:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177181.178855:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177181.178857:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178866:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177181.178868:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178872:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.178873:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177181.178874:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177181.178876:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177181.178878:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177181.178879:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302077064
00000100:00000001:3.0:1786177181.178881:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302080718
00000100:00000001:3.0:1786177181.178882:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302077064
00000100:00000001:3.0:1786177181.178883:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302080718
00000100:00000001:3.0:1786177181.178885:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177181.178886:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.178887:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177181.178889:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177181/real 1786177181]  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177181.178894:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177181.178894:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177181.178895:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177181.178897:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177181.178898:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177181.178899:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177181.178901:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177181.178906:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178910:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177181.178911:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.178913:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177181.178914:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177181.178928:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178933:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177181.178936:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178940:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177181.178942:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.178943:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177181.178944:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.178945:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177181.178946:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177181.178947:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177181.178949:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177181.178951:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177181.178953:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177181.178958:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177181.178960:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177181.178963:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.178967:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177181.178968:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177181.178969:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177181.178971:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177181.178972:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177181.178974:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177181.178976:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177181.178978:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177181.178979:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177181.178981:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177181.178983:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177181.178984:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177181.178987:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177181.178988:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177181.178991:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177181.178993:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856347136, portal 4
00000100:00000001:3.0:1786177181.178994:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177181.178995:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:3.0:1786177181.178999:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179003:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177181.179005:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177181.179006:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177181.179007:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856347136, offset 0
00000400:00000200:3.0:1786177181.179010:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177181.179012:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177181.179014:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177181.179016:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177181.179018:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177181.179019:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177181.179021:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177181.179023:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254883:254883:256:4294967295] 192.168.202.110@tcp LPNI seq info [254872:254872:8:4294967295]
00000400:00000200:3.0:1786177181.179026:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177181.179030:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177181.179032:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177181.179034:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177181.179037:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177181.179039:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786177181.179041:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177181.179043:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786177181.179046:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177181.179047:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177181.179049:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177181.179051:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177181.179053:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177181.179054:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177181.179056:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177181.179058:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179064:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177181.179065:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179069:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.179071:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177181.179072:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177181.179075:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177181.179077:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177181.179078:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302076939
00000100:00000001:3.0:1786177181.179080:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302080718
00000100:00000001:3.0:1786177181.179081:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302076939
00000100:00000001:3.0:1786177181.179083:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302080718
00000100:00000001:3.0:1786177181.179084:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177181.179085:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.179086:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177181.179088:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177181/real 1786177181]  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177181.179093:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177181.179094:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177181.179095:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177181.179097:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177181.179098:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177181.179099:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177181.179101:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177181.179106:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179110:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177181.179110:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179111:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177181.179112:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177181.179114:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179118:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177181.179120:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179124:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177181.179126:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179127:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177181.179127:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179129:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177181.179130:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177181.179131:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177181.179134:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177181.179135:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177181.179137:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177181.179146:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177181.179149:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179150:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177181.179152:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856346880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179158:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177181.179160:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177181.179161:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177181.179164:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177181.179165:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177181.179167:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177181.179169:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177181.179171:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:3.0:1786177181.179175:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177181.179177:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177181.179178:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:3.0:1786177181.179181:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177181.179183:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179185:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177181.179189:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856347008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179196:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177181.179197:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177181.179199:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177181.179201:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177181.179203:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177181.179204:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177181.179206:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177181.179208:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:3.0:1786177181.179210:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177181.179211:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177181.179212:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:3.0:1786177181.179215:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177181.179215:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179217:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177181.179219:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856347136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179223:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177181.179224:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177181.179225:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177181.179226:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177181.179227:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177181.179228:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177181.179229:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177181.179230:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:3.0:1786177181.179231:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177181.179232:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177181.179232:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:3.0:1786177181.179234:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177181.179234:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179235:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179238:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177181.179239:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.179241:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177181.179242:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177181.179243:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177181.179246:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179250:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177181.179251:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177181.179252:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177181.179253:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177181.179255:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177181.179256:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177181.179259:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177181.179260:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177181.179261:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177181.179263:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177181.179265:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177181.179267:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177181.179269:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177181.179271:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177181.179273:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177181.179275:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856347264, portal 4
00000100:00000001:3.0:1786177181.179277:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177181.179278:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:3.0:1786177181.179282:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179286:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177181.179288:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177181.179289:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177181.179291:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856347264, offset 0
00000400:00000200:3.0:1786177181.179293:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177181.179296:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177181.179298:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177181.179300:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177181.179301:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177181.179303:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177181.179304:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177181.179307:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254884:254884:256:4294967295] 192.168.202.110@tcp LPNI seq info [254873:254873:8:4294967295]
00000400:00000200:3.0:1786177181.179309:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177181.179314:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177181.179317:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177181.179319:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177181.179321:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177181.179323:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786177181.179326:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177181.179328:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786177181.179330:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177181.179332:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177181.179334:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177181.179336:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177181.179338:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177181.179340:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177181.179342:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177181.179346:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179354:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177181.179357:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179363:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.179365:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177181.179369:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177181.179372:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177181.179375:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177181.179378:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302076947
00000100:00000001:3.0:1786177181.179380:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302080718
00000100:00000001:3.0:1786177181.179383:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302076947
00000100:00000001:3.0:1786177181.179385:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302080718
00000100:00000001:3.0:1786177181.179387:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177181.179389:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.179392:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177181.179395:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177181/real 1786177181]  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177181.179401:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177181.179402:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177181.179403:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177181.179405:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177181.179406:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177181.179408:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177181.179410:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177181.179434:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179441:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177181.179443:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179444:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177181.179446:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177181.179450:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179458:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177181.179462:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179470:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177181.179472:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179474:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177181.179476:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179478:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177181.179479:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177181.179482:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177181.179486:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177181.179488:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177181.179492:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177181.179501:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177181.179505:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179507:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177181.179528:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856347264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177181.179533:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177181.179534:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177181.179536:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177181.179538:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177181.179539:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177181.179540:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177181.179541:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177181.179542:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:3.0:1786177181.179544:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177181.179545:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177181.179546:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:3.0:1786177181.179548:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177181.179548:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179550:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177181.179552:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177181.179553:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177181.179555:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177181.179556:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177181.752663:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177181.752674:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177181.752682:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177181.752683:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177181.752687:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177181.752689:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177181.752760:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177181.752765:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177181.752770:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177182.200635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177182.200645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177182.200647:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177182.200653:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177182.200656:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177182.200660:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177182.200663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177182.200665:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177182.200668:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177182.776641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177182.776651:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177182.776660:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177182.776662:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177182.776665:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177182.776666:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177182.776859:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177182.776865:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177182.776869:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.0:1786177183.096633:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786177183.096641:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786177183.096647:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1786177183.096850:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786177183.096870:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000200:1.0:1786177183.096873:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1786177183.096876:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786177183.096881:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177183.224658:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177183.224666:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177183.224668:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177183.224673:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177183.224675:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177183.224679:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177183.224681:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177183.224683:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177183.224685:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177183.800629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177183.800640:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177183.800644:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177183.800645:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177183.800648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177183.800650:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177183.800652:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177183.800654:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177183.800656:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177183.800657:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177183.800659:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177183.800660:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177183.800741:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177183.800747:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177183.800750:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177184.248627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177184.248636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177184.248639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177184.248644:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177184.248647:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177184.248651:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177184.248653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177184.248656:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177184.248659:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177184.824626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177184.824632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177184.824637:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177184.824638:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177184.824642:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177184.824643:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177184.824697:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177184.824701:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177184.824703:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177185.016610:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302080718, at 4302080768
00000100:00000040:1.0:1786177185.016610:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302080718, at 4302080768
00000100:00000010:0.0:1786177185.016626:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37ea0.
00000100:00000010:1.0:1786177185.016632:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37060.
00000100:00000001:3.0:1786177185.272626:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177185.272632:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177185.272633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177185.272637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177185.272640:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177185.272642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177185.272644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177185.272646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177185.272648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177185.848641:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177185.848649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177185.848656:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177185.848657:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177185.848661:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177185.848663:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177185.848721:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177185.848725:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177185.848728:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786177186.296637:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177186.296637:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177186.296646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786177186.296647:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177186.296652:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177186.296652:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177186.296657:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1786177186.296660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177186.296662:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177186.296667:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1786177186.296671:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7409
00000100:00000001:3.0:1786177186.296673:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.296678:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000400:00000200:0.0:1786177186.296681:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177186.296683:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1786177186.296689:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177186.296689:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177186.296693:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177186.296697:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.296700:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177186.296703:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.296705:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177186.296708:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177186.296710:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177186.296712:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177186.296715:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177186.296717:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177186.296719:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177186.296722:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.296724:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177186.296726:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.296728:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177186.296730:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177186.296732:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177186.296734:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177186.296736:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177186.296740:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177186.296742:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177186.296744:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177186.296745:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177186.296755:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419c00.
00000020:00000040:0.0:1786177186.296758:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177186.296762:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177186.296764:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177186.296766:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177186.296769:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177186.296773:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177186.296781:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000100:00000001:0.0:1786177186.296788:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177186.296797:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177186.296805:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177186.296809:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177186.296813:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cf890c1c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177186.296831:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177186.296833:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177186.296836:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177186.296837:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177186.296838:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177186.296840:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177186.296842:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7409
00000400:00000200:0.0:1786177186.296844:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177186.296846:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177186.296849:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177186.296851:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.296853:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177186.296855:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.296857:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177186.296859:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177186.296860:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177186.296862:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177186.296864:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177186.296865:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177186.296882:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177186.296885:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.296886:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177186.296887:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.296888:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177186.296890:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177186.296892:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177186.296893:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177186.296894:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177186.296898:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177186.296899:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177186.296903:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:0.0:1786177186.296904:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177186.296907:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177186.296909:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177186.296913:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177186.296916:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177186.296918:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177186.296921:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1786177186.296922:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41bb80.
00000020:00000040:0.0:1786177186.296926:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177186.296931:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177186.296934:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177186.296939:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177186.296944:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177186.296944:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177186.296948:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177186.296949:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177186.296954:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:0.0:1786177186.296957:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
02000000:00000001:3.0:1786177186.296959:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177186.296963:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177186.296965:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177186.296972:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1786177186.296974:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856347392:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177186.296984:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1786177186.296989:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1786177186.296990:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1786177186.296992:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177186.296996:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177186.296998:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9983a05] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177186.297002:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1786177186.297009:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177186.297011:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000010:3.0:1786177186.297011:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00080000:0.0:1786177186.297017:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
02000000:00000001:3.0:1786177186.297017:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177186.297019:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177186.297023:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177186.297025:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177186.297028:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7409
00000400:00000200:3.0:1786177186.297030:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177186.297034:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177186.297039:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000200:0.0:1786177186.297046:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177186.297048:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1786177186.297050:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000040:0.0:1786177186.297051:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297054:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177186.297057:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297059:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177186.297060:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177186.297061:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177186.297063:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000200:3.0:1786177186.297063:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856347392, portal 25
00000020:00000040:0.0:1786177186.297066:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177186.297069:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:3.0:1786177186.297069:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000020:00000001:0.0:1786177186.297073:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:3.0:1786177186.297073:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761661952 : -114547947889664 : ffff97d1ba419c00)
00000100:00000040:0.0:1786177186.297081:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297083:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177186.297087:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:3.0:1786177186.297088:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177186.297089:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177186.297094:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786177186.297098:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297101:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177186.297104:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177186.297107:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177186.297110:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.297110:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1786177186.297114:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177186.297115:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000040:3.0:1786177186.297119:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000008:00000001:0.0:1786177186.297124:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177186.297125:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000001:0.0:1786177186.297132:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177186.297133:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856347392, offset 0
00000100:00000010:0.0:1786177186.297136:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418700.
00000020:00000040:0.0:1786177186.297140:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177186.297142:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177186.297145:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177186.297147:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1786177186.297147:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786177186.297151:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177186.297154:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177186.297159:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000100:00000001:0.0:1786177186.297163:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177186.297169:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177186.297172:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177186.297183:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177186.297184:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1786177186.297187:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177186.297194:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1786177186.297196:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bf66f89d] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177186.297203:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1786177186.297212:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177186.297212:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:0.0:1786177186.297218:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177186.297224:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177186.297225:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177186.297227:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1786177186.297228:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297231:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177186.297237:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254885:254885:256:4294967295] 192.168.202.110@tcp LPNI seq info [254874:254874:8:4294967295]
00000100:00080000:0.0:1786177186.297239:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7409
00000400:00000200:0.0:1786177186.297248:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177186.297250:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177186.297255:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177186.297259:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177186.297265:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297271:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177186.297272:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1786177186.297277:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297281:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177186.297287:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000200:3.0:1786177186.297287:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:0.0:1786177186.297290:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177186.297293:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177186.297296:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177186.297298:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177186.297301:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000010:3.0:1786177186.297305:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000040:0.0:1786177186.297306:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297312:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177186.297314:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297318:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.0:1786177186.297320:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:0.0:1786177186.297323:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000800:00000200:3.0:1786177186.297328:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00080000:0.0:1786177186.297330:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297337:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177186.297341:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:0.0:1786177186.297342:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177186.297347:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:3.1:1786177186.297348:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
02000000:00000001:0.0:1786177186.297352:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177186.297357:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000800:00000100:3.0:1786177186.297358:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000008:00100000:0.0:1786177186.297361:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000800:00000010:3.0:1786177186.297364:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000008:00000001:0.0:1786177186.297367:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177186.297371:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177186.297373:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000010:0.0:1786177186.297376:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418000.
00000020:00000040:0.0:1786177186.297382:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000400:00000200:3.0:1786177186.297383:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177186.297393:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177186.297395:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
02000000:00000001:0.0:1786177186.297397:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177186.297401:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:0.0:1786177186.297405:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177186.297409:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177186.297410:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177186.297414:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177186.297417:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177186.297424:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000100:00000001:0.0:1786177186.297430:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177186.297436:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177186.297437:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177186.297446:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177186.297451:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177186.297456:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.297463:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177186.297465:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002e10e4f5] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.297469:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177186.297478:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.297479:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177186.297481:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177186.297486:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177186.297488:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1786177186.297488:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177186.297491:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177186.297495:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302085837
00000100:00000001:0.0:1786177186.297496:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177186.297501:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7409
00000400:00000200:0.0:1786177186.297505:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:3.0:1786177186.297506:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37660.
00000400:00000200:0.0:1786177186.297547:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177186.297550:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:0.0:1786177186.297551:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177186.297558:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297562:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177186.297562:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302085838
00000100:00000040:0.0:1786177186.297567:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000010:3.0:1786177186.297569:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37ae0.
00000100:00000001:0.0:1786177186.297571:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177186.297576:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000020:00000001:0.0:1786177186.297577:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177186.297580:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177186.297584:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177186.297584:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1786177186.297589:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:3.0:1786177186.297590:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177186.297595:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:3.0:1786177186.297596:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000020:00000001:0.0:1786177186.297601:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00100000:3.0:1786177186.297605:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177186/real 1786177186]  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177186.297607:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297611:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177186.297614:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297617:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177186.297622:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177186.297627:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177186.297630:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177186.297632:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177186.297635:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000001:3.0:1786177186.297635:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:0.0:1786177186.297640:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177186.297641:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1786177186.297642:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177186.297646:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000010:3.0:1786177186.297646:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000008:00000001:0.0:1786177186.297648:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177186.297651:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177186.297652:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000100:00000010:0.0:1786177186.297654:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b800.
00000400:00000010:3.0:1786177186.297657:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000020:00000040:0.0:1786177186.297658:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177186.297664:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177186.297664:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:0.0:1786177186.297668:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177186.297670:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177186.297671:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177186.297673:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177186.297674:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786177186.297677:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4c00.
00000100:00000001:0.0:1786177186.297682:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177186.297690:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1786177186.297695:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177186.297700:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177186.297703:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177186.297711:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000192ee9a0] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.297713:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177186.297717:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.297722:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177186.297723:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.297727:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177186.297729:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7419)
00000100:00100000:3.0:1786177186.297735:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.297745:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177186.297751:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.297762:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177186.297766:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.297768:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177186.297778:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.297781:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177186.297784:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177186.297788:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177186.297792:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177186.297795:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177186.297801:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177186.297811:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856347392:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177186.297816:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.297819:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177186.297823:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419c00 x1872934856347392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.297831:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177186.297834:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177186.297837:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177186.297840:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177186.297843:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177186.297845:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177186.297848:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177186.297851:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:3.0:1786177186.297855:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177186.297857:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177186.297859:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419c00.
00000100:00000001:3.0:1786177186.297863:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177186.297865:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.297880:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.297889:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177186.297890:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.297893:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177186.297894:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177186.297913:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177186.297918:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.297925:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177186.297926:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177186.297928:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177186.297930:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177186.297933:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177186.297936:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347520:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177186.297940:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177186.297942:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177186.297943:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177186.297946:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177186.297949:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177186.297952:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177186.297956:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177186.297958:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177186.297962:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177186.297965:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856347520, portal 10
00000100:00000001:3.0:1786177186.297967:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177186.297969:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761670016 : -114547947881600 : ffff97d1ba41bb80)
00000100:00000040:3.0:1786177186.297976:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.297982:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177186.297984:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177186.297986:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177186.297989:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856347520, offset 0
00000400:00000200:3.0:1786177186.297993:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177186.297997:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177186.298001:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177186.298004:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177186.298007:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177186.298009:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177186.298012:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177186.298015:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254886:254886:256:4294967295] 192.168.202.110@tcp LPNI seq info [254875:254875:8:4294967295]
00000400:00000200:3.0:1786177186.298019:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177186.298025:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177186.298030:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177186.298033:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177186.298037:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177186.298039:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786177186.298044:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177186.298047:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1786177186.298050:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177186.298052:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177186.298055:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177186.298058:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177186.298061:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177186.298063:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177186.298065:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177186.298069:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298077:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177186.298080:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298087:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.298089:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177186.298092:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177186.298096:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177186.298099:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177186.298102:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302082058
00000100:00000001:3.0:1786177186.298104:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302085837
00000100:00000001:3.0:1786177186.298107:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302082058
00000100:00000001:3.0:1786177186.298109:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302085838
00000100:00000001:3.0:1786177186.298112:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177186.298114:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.298116:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177186.298120:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177186/real 1786177186]  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177186.298129:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177186.298131:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177186.298133:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177186.298136:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177186.298138:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177186.298140:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177186.298144:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177186.298154:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298161:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177186.298163:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.298165:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177186.298166:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177186.298170:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298177:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177186.298180:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298185:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177186.298186:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.298187:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177186.298188:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.298189:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177186.298190:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177186.298192:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177186.298194:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177186.298196:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177186.298199:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177186.298204:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347520:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177186.298206:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177186.298208:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298217:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177186.298218:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177186.298220:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177186.298221:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177186.298223:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177186.298224:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177186.298226:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177186.298228:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177186.298229:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177186.298230:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177186.298233:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177186.298234:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177186.298237:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177186.298239:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177186.298241:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177186.298243:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856347648, portal 10
00000100:00000001:3.0:1786177186.298244:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177186.298246:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761656576 : -114547947895040 : ffff97d1ba418700)
00000100:00000040:3.0:1786177186.298250:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298254:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177186.298256:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177186.298258:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177186.298259:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856347648, offset 0
00000400:00000200:3.0:1786177186.298262:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177186.298265:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177186.298267:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177186.298269:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177186.298271:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177186.298273:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177186.298275:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177186.298277:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254887:254887:256:4294967295] 192.168.202.110@tcp LPNI seq info [254876:254876:8:4294967295]
00000400:00000200:3.0:1786177186.298280:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177186.298284:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177186.298287:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177186.298290:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177186.298294:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177186.298297:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786177186.298302:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177186.298305:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1786177186.298309:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177186.298311:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177186.298314:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177186.298318:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177186.298321:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177186.298323:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177186.298325:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177186.298329:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298336:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177186.298340:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298347:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.298348:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177186.298352:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177186.298354:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177186.298358:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177186.298361:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302082183
00000100:00000001:3.0:1786177186.298364:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302085837
00000100:00000001:3.0:1786177186.298366:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302082183
00000100:00000001:3.0:1786177186.298368:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302085838
00000100:00000001:3.0:1786177186.298371:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177186.298374:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.298376:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177186.298380:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177186/real 1786177186]  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177186.298388:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177186.298390:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177186.298392:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177186.298394:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177186.298396:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177186.298398:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177186.298401:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177186.298409:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298416:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177186.298418:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.298419:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177186.298421:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177186.298424:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298431:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177186.298434:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298441:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177186.298444:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.298445:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177186.298446:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.298448:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177186.298450:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177186.298452:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177186.298456:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177186.298458:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177186.298462:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177186.298470:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177186.298474:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177186.298477:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298484:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177186.298486:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177186.298488:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177186.298490:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177186.298492:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177186.298495:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177186.298499:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177186.298501:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177186.298502:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177186.298504:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177186.298526:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177186.298529:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177186.298534:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177186.298537:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177186.298540:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177186.298544:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856347776, portal 4
00000100:00000001:3.0:1786177186.298546:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177186.298547:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761654784 : -114547947896832 : ffff97d1ba418000)
00000100:00000040:3.0:1786177186.298555:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298562:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177186.298565:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177186.298567:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177186.298570:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856347776, offset 0
00000400:00000200:3.0:1786177186.298574:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177186.298578:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177186.298582:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177186.298585:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177186.298588:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177186.298591:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177186.298593:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177186.298597:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254888:254888:256:4294967295] 192.168.202.110@tcp LPNI seq info [254877:254877:8:4294967295]
00000400:00000200:3.0:1786177186.298601:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177186.298608:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177186.298613:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177186.298616:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177186.298620:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177186.298623:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786177186.298628:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177186.298631:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1786177186.298635:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177186.298637:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177186.298640:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177186.298644:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177186.298647:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177186.298649:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177186.298652:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177186.298656:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298663:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177186.298665:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298670:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.298671:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177186.298674:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177186.298677:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177186.298680:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177186.298681:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302082059
00000100:00000001:3.0:1786177186.298683:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302085837
00000100:00000001:3.0:1786177186.298685:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302082059
00000100:00000001:3.0:1786177186.298687:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302085838
00000100:00000001:3.0:1786177186.298688:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177186.298690:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.298691:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177186.298693:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177186/real 1786177186]  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177186.298698:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177186.298699:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177186.298700:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177186.298701:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177186.298702:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177186.298703:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177186.298705:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177186.298711:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298715:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177186.298716:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.298717:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177186.298718:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177186.298720:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298724:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177186.298727:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298732:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177186.298734:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.298735:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177186.298736:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.298738:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177186.298739:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177186.298742:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177186.298745:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177186.298747:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177186.298752:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177186.298761:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177186.298765:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177186.298769:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298776:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177186.298777:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177186.298780:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177186.298782:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177186.298784:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177186.298787:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347904:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177186.298791:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177186.298793:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177186.298795:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177186.298796:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177186.298800:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177186.298802:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177186.298806:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177186.298809:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177186.298812:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177186.298816:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856347904, portal 4
00000100:00000001:3.0:1786177186.298818:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177186.298819:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761669120 : -114547947882496 : ffff97d1ba41b800)
00000100:00000040:3.0:1786177186.298825:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298832:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177186.298835:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177186.298838:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177186.298840:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856347904, offset 0
00000400:00000200:3.0:1786177186.298845:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177186.298849:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177186.298853:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177186.298856:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177186.298859:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177186.298862:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177186.298864:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177186.298883:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254889:254889:256:4294967295] 192.168.202.110@tcp LPNI seq info [254878:254878:8:4294967295]
00000400:00000200:3.0:1786177186.298888:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177186.298895:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177186.298900:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177186.298904:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177186.298908:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177186.298911:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1786177186.298916:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177186.298919:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1786177186.298923:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177186.298925:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177186.298928:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177186.298932:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177186.298935:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177186.298937:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177186.298940:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177186.298944:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298953:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177186.298956:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.298962:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.298964:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177186.298968:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177186.298971:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177186.298975:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177186.298977:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302082067
00000100:00000001:3.0:1786177186.298980:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302085837
00000100:00000001:3.0:1786177186.298983:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302082067
00000100:00000001:3.0:1786177186.298985:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302085838
00000100:00000001:3.0:1786177186.298987:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177186.298989:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.298991:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177186.298996:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177186/real 1786177186]  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177186.299004:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177186.299006:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177186.299008:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177186.299010:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177186.299012:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177186.299015:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177186.299019:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177186.299028:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.299042:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177186.299044:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.299046:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177186.299047:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177186.299051:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.299059:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177186.299063:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.299071:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177186.299074:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.299076:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177186.299077:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.299079:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177186.299081:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177186.299084:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177186.299087:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177186.299090:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177186.299094:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177186.299103:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856347904:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177186.299107:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.299109:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177186.299112:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41bb80 x1872934856347520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.299119:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177186.299121:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177186.299123:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177186.299126:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177186.299128:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177186.299129:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177186.299131:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177186.299133:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
02000000:00000001:3.0:1786177186.299136:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177186.299137:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177186.299139:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41bb80.
00000100:00000001:3.0:1786177186.299142:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177186.299143:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.299145:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177186.299148:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418700 x1872934856347648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.299155:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177186.299156:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177186.299157:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177186.299159:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177186.299160:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177186.299161:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177186.299163:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177186.299165:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:3.0:1786177186.299167:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177186.299168:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177186.299169:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418700.
00000100:00000001:3.0:1786177186.299173:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177186.299175:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.299176:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177186.299181:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418000 x1872934856347776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.299188:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177186.299189:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177186.299191:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177186.299194:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177186.299195:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177186.299197:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177186.299200:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177186.299201:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:3.0:1786177186.299204:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177186.299205:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177186.299206:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418000.
00000100:00000001:3.0:1786177186.299209:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177186.299210:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.299212:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177186.299216:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b800 x1872934856347904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177186.299223:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177186.299224:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177186.299226:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177186.299228:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177186.299230:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177186.299231:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177186.299233:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177186.299235:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:3.0:1786177186.299237:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177186.299238:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177186.299239:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b800.
00000100:00000001:3.0:1786177186.299242:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177186.299243:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.299246:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177186.299250:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177186.299251:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177186.299254:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177186.299255:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177186.872613:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177186.872618:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177186.872622:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177186.872623:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177186.872625:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177186.872626:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177186.872679:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177186.872683:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177186.872687:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177187.320649:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177187.320656:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177187.320657:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177187.320662:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177187.320664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177187.320667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177187.320669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177187.320671:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177187.320673:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177187.896580:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177187.896584:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177187.896588:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177187.896589:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177187.896592:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177187.896592:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177187.896649:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177187.896651:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177187.896653:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177188.344628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177188.344640:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177188.344643:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177188.344650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177188.344654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177188.344661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177188.344664:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177188.344666:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177188.344669:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177188.920628:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177188.920635:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177188.920639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177188.920641:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177188.920643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177188.920645:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177188.920690:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177188.920693:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177188.920696:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177189.368610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177189.368616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177189.368617:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177189.368622:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177189.368624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177189.368627:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177189.368630:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177189.368631:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177189.368633:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177189.944616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177189.944620:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177189.944624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177189.944625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177189.944627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177189.944628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177189.944683:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177189.944686:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177189.944688:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177190.136611:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302085837, at 4302085888
00000100:00000040:1.0:1786177190.136611:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302085838, at 4302085888
00000100:00000010:1.0:1786177190.136626:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37ae0.
00000100:00000010:0.0:1786177190.136629:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37660.
00000100:00000001:3.0:1786177190.392594:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177190.392600:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177190.392601:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177190.392604:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177190.392607:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177190.392609:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177190.392612:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177190.392614:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177190.392615:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177190.968617:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177190.968623:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177190.968624:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177190.968627:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177190.968630:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177190.968633:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177190.968634:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177190.968636:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177190.968639:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177190.968640:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177190.968641:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177190.968642:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177190.968644:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177190.968645:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177190.968647:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177190.968648:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177190.968651:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177190.968653:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177190.968697:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177190.968700:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177190.968702:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177191.416585:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:0.0:1786177191.416587:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177191.416590:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177191.416593:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177191.416594:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:3.0:1786177191.416597:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177191.416598:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177191.416601:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177191.416602:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177191.416606:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177191.416607:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177191.416611:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1786177191.416615:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7414
00000100:00000001:3.0:1786177191.416615:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.416620:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177191.416625:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177191.416631:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177191.416634:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177191.416636:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177191.416639:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177191.416641:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177191.416644:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177191.416647:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177191.416649:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177191.416652:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177191.416654:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177191.416656:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177191.416661:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177191.416663:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177191.416665:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177191.416666:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177191.416668:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177191.416671:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177191.416673:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177191.416676:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177191.416678:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177191.416681:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177191.416684:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177191.416685:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177191.416687:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177191.416695:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419f80.
00000020:00000040:0.0:1786177191.416706:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177191.416710:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177191.416711:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177191.416714:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177191.416717:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177191.416721:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177191.416727:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000100:00000001:0.0:1786177191.416734:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177191.416742:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177191.416751:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177191.416755:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177191.416759:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ad753ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177191.416777:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177191.416780:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177191.416783:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177191.416784:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177191.416786:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177191.416789:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177191.416792:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7414
00000100:00000001:3.0:1786177191.416796:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1786177191.416798:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177191.416800:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177191.416802:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000400:00000200:0.0:1786177191.416803:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177191.416807:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177191.416809:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177191.416813:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177191.416834:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177191.416835:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177191.416837:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177191.416839:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177191.416841:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177191.416844:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856348032:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177191.416849:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177191.416849:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177191.416855:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:0.0:1786177191.416856:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177191.416860:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177191.416860:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177191.416865:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:0.0:1786177191.416866:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177191.416870:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177191.416872:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177191.416874:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
02000000:00000010:3.0:1786177191.416874:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
00000020:00000001:0.0:1786177191.416876:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:3.0:1786177191.416879:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177191.416881:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177191.416886:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177191.416888:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177191.416891:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1786177191.416895:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000040:0.0:1786177191.416896:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177191.416901:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177191.416903:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177191.416905:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:3.0:1786177191.416906:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177191.416907:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177191.416910:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1786177191.416914:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856348032, portal 25
02000000:00000001:0.0:1786177191.416915:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177191.416919:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.416920:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177191.416921:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177191.416922:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761662848 : -114547947888768 : ffff97d1ba419f80)
00000008:00100000:0.0:1786177191.416925:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177191.416927:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177191.416930:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177191.416931:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1786177191.416934:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418380.
00000020:00000040:0.0:1786177191.416938:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177191.416943:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177191.416943:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1786177191.416947:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177191.416949:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177191.416951:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1786177191.416953:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
02000000:00000001:0.0:1786177191.416954:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:3.0:1786177191.416958:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856348032, offset 0
02000000:00000001:0.0:1786177191.416959:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177191.416967:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
00000400:00000200:3.0:1786177191.416967:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177191.416974:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177191.416978:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:0.0:1786177191.416985:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177191.416986:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177191.416992:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177191.416998:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1786177191.416999:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177191.417002:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1786177191.417003:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177191.417008:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:0.0:1786177191.417011:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e6b901d7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177191.417015:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254890:254890:256:4294967295] 192.168.202.110@tcp LPNI seq info [254879:254879:8:4294967295]
00000400:00000200:3.0:1786177191.417025:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177191.417030:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177191.417033:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177191.417036:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177191.417037:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177191.417038:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177191.417040:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177191.417044:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7414
00000400:00000200:3.0:1786177191.417049:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:0.0:1786177191.417056:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:3.0:1786177191.417057:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177191.417062:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000010:3.0:1786177191.417063:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00080000:0.0:1786177191.417065:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177191.417068:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177191.417072:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1786177191.417073:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786177191.417076:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:3.0:1786177191.417078:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:0.0:1786177191.417079:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177191.417083:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177191.417085:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177191.417087:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:3.1:1786177191.417088:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:0.0:1786177191.417089:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000800:00000200:3.1:1786177191.417093:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:0.0:1786177191.417094:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177191.417097:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000100:3.0:1786177191.417101:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:0.0:1786177191.417103:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000010:3.0:1786177191.417104:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00000001:0.0:1786177191.417106:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177191.417109:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177191.417110:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1786177191.417112:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177191.417115:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:3.0:1786177191.417117:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:0.0:1786177191.417120:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177191.417122:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177191.417123:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
02000000:00000001:0.0:1786177191.417124:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1786177191.417127:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
02000000:00000001:0.0:1786177191.417128:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177191.417132:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177191.417133:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177191.417134:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000008:00100000:0.0:1786177191.417135:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177191.417137:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177191.417140:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177191.417140:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1786177191.417146:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b800.
00000020:00000040:0.0:1786177191.417150:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000001:3.0:1786177191.417151:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1786177191.417152:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177191.417154:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177191.417155:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177191.417156:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177191.417158:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177191.417161:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177191.417166:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000100:00000001:3.0:1786177191.417166:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.417170:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177191.417171:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.417175:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177191.417176:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177191.417182:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177191.417186:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302090957
00000100:00000001:0.0:1786177191.417189:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177191.417191:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177191.417193:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37660.
00000100:00000040:0.0:1786177191.417195:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000192ee9a0] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417200:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786177191.417201:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177191.417204:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177191.417206:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302090957
00000100:00080000:0.0:1786177191.417207:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177191.417210:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177191.417212:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000010:3.0:1786177191.417212:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37ae0.
00000100:00000001:3.0:1786177191.417216:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786177191.417217:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177191.417221:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7414
00000400:00000200:0.0:1786177191.417223:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177191.417225:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177191.417227:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177191.417227:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177191.417232:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177191.417233:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177191.417236:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1786177191.417237:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177191.417240:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177191.417242:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177191.417244:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00100000:3.0:1786177191.417244:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177191/real 1786177191]  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177191.417247:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177191.417250:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177191.417252:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177191.417256:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000001:3.0:1786177191.417258:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:0.0:1786177191.417259:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000400:00000001:3.0:1786177191.417262:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:0.0:1786177191.417265:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:3.0:1786177191.417266:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00000001:0.0:1786177191.417267:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177191.417271:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177191.417271:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000100:00000001:0.0:1786177191.417274:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000010:3.0:1786177191.417275:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000020:00000040:0.0:1786177191.417277:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:3.0:1786177191.417278:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00080000:0.0:1786177191.417281:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:3.0:1786177191.417284:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177191.417285:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177191.417289:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177191.417292:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177191.417296:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177191.417299:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000200:3.0:1786177191.417300:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1786177191.417301:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177191.417306:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177191.417309:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177191.417315:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418000.
00000100:00000001:3.0:1786177191.417315:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177191.417317:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1786177191.417319:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177191.417320:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:0.0:1786177191.417323:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177191.417324:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177191.417325:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177191.417329:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:3.0:1786177191.417329:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177191.417332:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177191.417334:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177191.417338:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000100:00000001:3.0:1786177191.417340:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177191.417342:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177191.417345:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177191.417349:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177191.417362:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177191.417365:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177191.417366:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177191.417368:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.417370:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177191.417372:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177191.417373:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002e10e4f5] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417374:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177191.417377:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177191.417382:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:0.0:1786177191.417387:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177191.417390:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00080000:0.0:1786177191.417391:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177191.417394:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00080000:0.0:1786177191.417395:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177191.417398:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:3.0:1786177191.417400:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177191.417401:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177191.417406:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177191.417410:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7414
00000100:00100000:3.0:1786177191.417415:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856348032:192.168.202.110@tcp:250:kworker.0
00000400:00000200:0.0:1786177191.417416:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177191.417421:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177191.417422:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177191.417423:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177191.417425:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177191.417427:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177191.417431:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177191.417431:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419f80 x1872934856348032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177191.417436:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177191.417440:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177191.417444:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177191.417446:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:3.0:1786177191.417446:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000020:00000001:0.0:1786177191.417449:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
02000000:00000001:3.0:1786177191.417449:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000020:00000040:0.0:1786177191.417453:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
02000000:00000010:3.0:1786177191.417453:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
00000020:00000001:0.0:1786177191.417456:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
02000000:00000001:3.0:1786177191.417458:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177191.417459:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177191.417461:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000040:0.0:1786177191.417463:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000020:00000040:3.0:1786177191.417464:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:0.0:1786177191.417466:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000020:00000001:3.0:1786177191.417467:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00000040:0.0:1786177191.417469:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177191.417472:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000010:3.0:1786177191.417472:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
00000020:00000040:0.0:1786177191.417477:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
02000000:00000001:3.0:1786177191.417477:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00080000:0.0:1786177191.417479:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1786177191.417480:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177191.417484:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419f80.
00000100:00000001:0.0:1786177191.417485:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177191.417488:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177191.417490:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
02000000:00000001:0.0:1786177191.417491:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177191.417493:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177191.417494:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177191.417497:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177191.417497:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:0.0:1786177191.417499:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177191.417502:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177191.417503:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177191.417505:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418700.
00000100:00000001:3.0:1786177191.417505:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000020:00000040:0.0:1786177191.417508:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177191.417525:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.417528:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177191.417530:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
02000000:00000001:0.0:1786177191.417531:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177191.417531:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:0.0:1786177191.417534:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177191.417535:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177191.417536:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177191.417538:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177191.417541:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.417546:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000010:0.0:1786177191.417547:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6000.
00000100:00000001:3.0:1786177191.417548:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177191.417550:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177191.417551:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177191.417554:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00100000:0.0:1786177191.417558:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1786177191.417558:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177191.417564:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856348160:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177191.417571:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177191.417571:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1786177191.417575:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177191.417575:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177191.417578:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177191.417580:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:0.0:1786177191.417581:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bf66f89d] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1786177191.417584:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177191.417588:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177191.417595:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177191.417595:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177191.417600:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000040:0.0:1786177191.417602:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7424)
00000400:00000010:3.0:1786177191.417604:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177191.417607:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856348160, portal 10
00000100:00000001:3.0:1786177191.417609:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177191.417609:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761655680 : -114547947895936 : ffff97d1ba418380)
00000100:00000040:3.0:1786177191.417616:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417620:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177191.417621:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177191.417623:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177191.417624:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856348160, offset 0
00000400:00000200:3.0:1786177191.417626:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177191.417628:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177191.417630:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177191.417632:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177191.417634:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177191.417635:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177191.417636:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177191.417638:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254891:254891:256:4294967295] 192.168.202.110@tcp LPNI seq info [254880:254880:8:4294967295]
00000400:00000200:3.0:1786177191.417641:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177191.417644:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177191.417647:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177191.417649:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177191.417652:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177191.417653:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786177191.417656:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177191.417657:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1786177191.417660:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177191.417661:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177191.417663:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177191.417665:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177191.417667:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177191.417668:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177191.417669:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177191.417671:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417676:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177191.417677:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417681:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.417682:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177191.417684:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177191.417687:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177191.417688:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177191.417689:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302087178
00000100:00000001:3.0:1786177191.417691:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302090957
00000100:00000001:3.0:1786177191.417692:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302087178
00000100:00000001:3.0:1786177191.417694:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302090957
00000100:00000001:3.0:1786177191.417695:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177191.417696:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.417697:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177191.417699:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177191/real 1786177191]  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177191.417704:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177191.417705:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177191.417706:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177191.417707:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177191.417708:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177191.417709:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177191.417710:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177191.417715:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417718:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177191.417719:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.417720:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177191.417721:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177191.417723:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417727:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177191.417729:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417732:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177191.417733:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.417734:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177191.417735:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.417736:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177191.417737:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177191.417738:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177191.417740:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177191.417741:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177191.417744:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177191.417748:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856348160:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177191.417751:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177191.417752:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417756:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177191.417756:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177191.417758:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177191.417758:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177191.417759:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177191.417761:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856348288:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177191.417762:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177191.417763:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177191.417764:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177191.417765:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177191.417767:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177191.417768:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177191.417770:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177191.417771:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177191.417773:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177191.417775:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856348288, portal 10
00000100:00000001:3.0:1786177191.417776:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177191.417777:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761669120 : -114547947882496 : ffff97d1ba41b800)
00000100:00000040:3.0:1786177191.417780:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417783:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177191.417785:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177191.417786:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177191.417787:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856348288, offset 0
00000400:00000200:3.0:1786177191.417789:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177191.417791:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177191.417793:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177191.417794:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177191.417796:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177191.417797:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177191.417799:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177191.417800:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254892:254892:256:4294967295] 192.168.202.110@tcp LPNI seq info [254881:254881:8:4294967295]
00000400:00000200:3.0:1786177191.417803:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177191.417806:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177191.417808:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177191.417809:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177191.417811:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177191.417813:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786177191.417815:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177191.417828:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1786177191.417832:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177191.417833:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177191.417835:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177191.417837:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177191.417839:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177191.417840:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177191.417842:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177191.417844:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417848:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177191.417850:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417854:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.417855:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177191.417857:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177191.417859:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177191.417862:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177191.417864:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302087303
00000100:00000001:3.0:1786177191.417866:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302090957
00000100:00000001:3.0:1786177191.417868:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302087303
00000100:00000001:3.0:1786177191.417870:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302090957
00000100:00000001:3.0:1786177191.417872:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177191.417874:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.417875:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177191.417879:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177191/real 1786177191]  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177191.417886:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177191.417888:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177191.417889:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177191.417891:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177191.417893:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177191.417895:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177191.417898:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177191.417905:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417909:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177191.417910:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.417911:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177191.417912:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177191.417914:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417918:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177191.417920:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417923:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177191.417925:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.417926:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177191.417927:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.417928:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177191.417929:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177191.417931:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177191.417933:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177191.417934:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177191.417936:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177191.417940:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856348288:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177191.417942:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177191.417944:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417947:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177191.417948:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177191.417949:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177191.417950:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177191.417951:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177191.417953:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856348416:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177191.417955:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177191.417956:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177191.417957:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177191.417958:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177191.417961:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177191.417962:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177191.417964:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177191.417965:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177191.417967:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177191.417969:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856348416, portal 4
00000100:00000001:3.0:1786177191.417970:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177191.417971:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761654784 : -114547947896832 : ffff97d1ba418000)
00000100:00000040:3.0:1786177191.417974:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.417978:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177191.417979:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177191.417981:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177191.417982:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856348416, offset 0
00000400:00000200:3.0:1786177191.417984:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177191.417986:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177191.417988:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177191.417990:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177191.417992:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177191.417993:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177191.417995:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177191.417996:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254893:254893:256:4294967295] 192.168.202.110@tcp LPNI seq info [254882:254882:8:4294967295]
00000400:00000200:3.0:1786177191.417999:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177191.418002:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177191.418005:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177191.418006:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177191.418009:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177191.418010:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786177191.418012:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177191.418014:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1786177191.418016:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177191.418017:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177191.418019:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177191.418021:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177191.418023:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177191.418024:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177191.418025:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177191.418027:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418035:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177191.418037:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418041:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.418042:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177191.418043:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177191.418045:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177191.418047:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177191.418048:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302087178
00000100:00000001:3.0:1786177191.418049:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302090957
00000100:00000001:3.0:1786177191.418050:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302087178
00000100:00000001:3.0:1786177191.418051:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302090957
00000100:00000001:3.0:1786177191.418053:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177191.418054:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.418055:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177191.418057:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177191/real 1786177191]  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177191.418060:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177191.418061:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177191.418062:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177191.418063:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177191.418064:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177191.418065:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177191.418067:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177191.418071:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418075:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177191.418076:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418077:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177191.418077:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177191.418079:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418083:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177191.418085:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418088:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177191.418089:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418090:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177191.418091:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418092:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177191.418093:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177191.418094:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177191.418095:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177191.418096:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177191.418098:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177191.418102:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856348416:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177191.418104:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418105:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177191.418107:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418380 x1872934856348160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418111:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177191.418112:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177191.418113:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177191.418114:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177191.418115:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177191.418116:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177191.418118:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177191.418119:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
02000000:00000001:3.0:1786177191.418120:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177191.418121:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177191.418122:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418380.
00000100:00000001:3.0:1786177191.418124:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177191.418125:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418126:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177191.418128:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b800 x1872934856348288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418131:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177191.418132:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177191.418133:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177191.418134:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177191.418135:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177191.418135:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177191.418136:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177191.418137:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:3.0:1786177191.418138:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177191.418139:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177191.418139:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b800.
00000100:00000001:3.0:1786177191.418140:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177191.418141:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418142:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177191.418143:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418000 x1872934856348416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418147:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177191.418147:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177191.418148:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177191.418149:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177191.418150:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177191.418150:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177191.418151:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177191.418152:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:3.0:1786177191.418153:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177191.418154:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177191.418155:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418000.
00000100:00000001:3.0:1786177191.418156:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177191.418157:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418158:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418160:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177191.418161:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.418162:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177191.418162:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177191.418163:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177191.418165:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418169:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177191.418170:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177191.418172:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177191.418173:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177191.418174:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177191.418175:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856348544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177191.418177:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177191.418177:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177191.418178:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177191.418179:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177191.418180:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177191.418181:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177191.418183:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177191.418184:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177191.418186:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177191.418187:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856348544, portal 4
00000100:00000001:3.0:1786177191.418188:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177191.418189:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761656576 : -114547947895040 : ffff97d1ba418700)
00000100:00000040:3.0:1786177191.418192:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418196:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177191.418197:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177191.418198:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177191.418200:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856348544, offset 0
00000400:00000200:3.0:1786177191.418202:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177191.418204:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177191.418206:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177191.418207:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177191.418209:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177191.418210:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177191.418212:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177191.418214:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254894:254894:256:4294967295] 192.168.202.110@tcp LPNI seq info [254883:254883:8:4294967295]
00000400:00000200:3.0:1786177191.418216:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177191.418219:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177191.418221:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177191.418223:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177191.418225:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177191.418226:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1786177191.418229:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177191.418230:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1786177191.418233:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177191.418234:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177191.418235:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177191.418237:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177191.418239:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177191.418240:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177191.418241:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177191.418243:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418247:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177191.418249:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418252:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.418253:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177191.418255:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177191.418257:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177191.418259:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177191.418260:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302087186
00000100:00000001:3.0:1786177191.418261:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302090957
00000100:00000001:3.0:1786177191.418262:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302087186
00000100:00000001:3.0:1786177191.418264:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302090957
00000100:00000001:3.0:1786177191.418265:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177191.418266:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.418267:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177191.418268:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177191/real 1786177191]  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177191.418272:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177191.418273:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177191.418273:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177191.418275:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177191.418276:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177191.418277:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177191.418278:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177191.418283:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418286:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177191.418287:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418288:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177191.418288:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177191.418290:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418294:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177191.418296:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418300:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177191.418301:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418302:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177191.418303:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418304:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177191.418305:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177191.418306:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177191.418308:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177191.418309:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177191.418311:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177191.418315:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856348544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177191.418316:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418317:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177191.418319:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418700 x1872934856348544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177191.418322:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177191.418323:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177191.418324:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177191.418325:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177191.418325:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177191.418326:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177191.418327:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177191.418328:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6000.
02000000:00000001:3.0:1786177191.418329:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177191.418330:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177191.418331:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418700.
00000100:00000001:3.0:1786177191.418332:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177191.418333:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418334:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177191.418335:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177191.418336:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177191.418337:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177191.418338:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177191.992614:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177191.992620:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177191.992626:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177191.992628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177191.992631:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177191.992632:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177191.992686:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177191.992690:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177191.992694:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177192.440602:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177192.440608:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177192.440609:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177192.440613:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177192.440616:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177192.440618:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177192.440620:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177192.440622:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177192.440625:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177193.016602:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177193.016609:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177193.016615:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177193.016616:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177193.016620:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177193.016621:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177193.016667:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177193.016670:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177193.016672:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177193.464588:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177193.464594:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177193.464596:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177193.464602:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177193.464605:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177193.464609:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177193.464611:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177193.464614:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177193.464617:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177194.040610:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177194.040616:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177194.040624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177194.040625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177194.040629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177194.040632:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177194.040634:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177194.040637:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177194.040641:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177194.040642:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177194.040644:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177194.040645:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177194.040692:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177194.040695:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177194.040699:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177194.488586:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177194.488593:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177194.488595:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177194.488599:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177194.488601:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177194.488604:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177194.488606:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177194.488609:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177194.488610:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177195.064603:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177195.064608:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177195.064612:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177195.064614:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177195.064616:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177195.064617:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177195.064669:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177195.064674:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177195.064678:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177195.256598:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302090957, at 4302091008
00000100:00000010:0.0:1786177195.256612:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37660.
00000100:00000040:1.0:1786177195.256618:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302090957, at 4302091008
00000100:00000010:1.0:1786177195.256626:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37ae0.
00000100:00000001:3.0:1786177195.512649:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177195.512657:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177195.512659:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177195.512665:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177195.512670:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177195.512673:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177195.512677:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177195.512681:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177195.512684:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177196.088667:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177196.088675:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177196.088680:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177196.088681:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177196.088684:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177196.088685:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177196.088756:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177196.088761:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177196.088765:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177196.536642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177196.536652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177196.536654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177196.536662:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786177196.536663:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177196.536668:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177196.536674:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177196.536675:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177196.536687:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177196.536687:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1786177196.536697:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177196.536697:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177196.536703:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177196.536706:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177196.536712:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7419
00000400:00000200:2.0:1786177196.536721:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177196.536727:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177196.536730:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177196.536732:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.536735:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177196.536737:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.536739:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177196.536742:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177196.536743:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177196.536746:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177196.536749:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177196.536751:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177196.536753:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177196.536755:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.536756:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177196.536758:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.536759:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177196.536762:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177196.536764:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177196.536765:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177196.536779:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177196.536782:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177196.536784:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177196.536786:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177196.536788:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177196.536795:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000020:00000040:2.0:1786177196.536798:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177196.536801:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177196.536802:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177196.536805:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177196.536808:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177196.536811:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177196.536824:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786177196.536829:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177196.536836:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177196.536848:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177196.536852:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177196.536855:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177196.536874:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177196.536876:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177196.536879:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177196.536880:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177196.536881:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177196.536884:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177196.536886:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7419
00000400:00000200:2.0:1786177196.536891:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177196.536893:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177196.536895:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177196.536897:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.536900:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177196.536900:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1786177196.536905:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177196.536907:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177196.536909:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177196.536911:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1786177196.536913:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177196.536915:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177196.536916:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177196.536918:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177196.536926:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177196.536927:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177196.536935:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177196.536946:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.536952:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177196.536955:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.536962:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177196.536969:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177196.536978:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177196.536986:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.536986:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1786177196.536998:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177196.536999:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177196.537011:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177196.537012:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177196.537020:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177196.537021:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177196.537037:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:3.0:1786177196.537037:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1786177196.537048:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177196.537055:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177196.537055:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856348672:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177196.537061:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177196.537071:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000100:00000001:3.0:1786177196.537072:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1786177196.537076:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177196.537081:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177196.537082:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177196.537086:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177196.537088:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177196.537090:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177196.537094:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1786177196.537095:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177196.537102:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177196.537108:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
02000000:00000010:3.0:1786177196.537109:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:2.0:1786177196.537114:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177196.537115:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177196.537132:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1786177196.537134:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1786177196.537138:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000001:2.0:1786177196.537148:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000010:3.0:1786177196.537150:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:2.0:1786177196.537151:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177196.537166:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177196.537169:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856348672, portal 25
00000100:00000001:3.0:1786177196.537182:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177196.537187:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00000040:3.0:1786177196.537197:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177196.537205:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.537206:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177196.537212:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177196.537215:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00080000:2.0:1786177196.537216:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:3.0:1786177196.537220:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856348672, offset 0
00000100:00080000:2.0:1786177196.537223:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177196.537226:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1786177196.537230:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786177196.537231:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177196.537237:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177196.537241:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7419
00000400:00000200:3.0:1786177196.537246:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177196.537259:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177196.537266:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177196.537271:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177196.537276:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177196.537280:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177196.537284:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254895:254895:256:4294967295] 192.168.202.110@tcp LPNI seq info [254884:254884:8:4294967295]
00000400:00000200:3.0:1786177196.537288:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177196.537294:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177196.537304:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177196.537311:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1786177196.537312:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1786177196.537322:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.537328:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1786177196.537329:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177196.537337:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000010:3.0:1786177196.537338:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000001:2.0:1786177196.537340:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177196.537345:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177196.537347:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177196.537349:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177196.537353:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000800:00000200:3.0:1786177196.537353:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786177196.537364:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:3.0:1786177196.537365:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:2.0:1786177196.537373:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000200:3.1:1786177196.537379:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:2.0:1786177196.537380:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:3.1:1786177196.537395:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:2.0:1786177196.537396:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:3.0:1786177196.537415:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177196.537416:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000010:3.0:1786177196.537418:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000100:00000001:2.0:1786177196.537426:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177196.537428:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:2.0:1786177196.537435:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:3.0:1786177196.537438:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177196.537450:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000100:00080000:2.0:1786177196.537456:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:3.0:1786177196.537457:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:2.0:1786177196.537469:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.537472:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1786177196.537475:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177196.537481:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:3.0:1786177196.537482:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177196.537497:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.537499:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786177196.537502:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:3.0:1786177196.537508:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1786177196.537509:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1786177196.537547:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.537549:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177196.537558:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177196.537564:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177196.537567:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302096078
00000100:00000010:3.0:1786177196.537577:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37060.
00000008:00000001:2.0:1786177196.537581:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177196.537586:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.537590:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000010:2.0:1786177196.537593:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000020:00000040:2.0:1786177196.537598:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177196.537602:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177196.537602:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302096078
02000000:00000001:2.0:1786177196.537605:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177196.537610:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177196.537613:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000010:3.0:1786177196.537613:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37ea0.
02000000:00000001:2.0:1786177196.537622:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.537624:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000010:2.0:1786177196.537629:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:3.0:1786177196.537631:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177196.537634:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.537635:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.537645:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177196.537650:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1786177196.537656:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177196/real 1786177196]  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177196.537690:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177196.537697:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177196.537708:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177196.537712:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786177196.537720:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177196.537720:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177196.537728:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00080000:2.0:1786177196.537729:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177196.537739:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177196.537741:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1786177196.537741:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000100:00080000:2.0:1786177196.537744:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:3.0:1786177196.537745:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:2.0:1786177196.537750:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177196.537752:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00080000:2.0:1786177196.537756:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7419
00000100:00000200:3.0:1786177196.537762:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786177196.537786:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177196.537802:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177196.537806:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177196.537819:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:3.0:1786177196.537819:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177196.537828:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177196.537840:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.537849:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177196.537855:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177196.537855:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000040:2.0:1786177196.537866:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:3.0:1786177196.537867:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1786177196.537875:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177196.537875:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177196.537880:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786177196.537881:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177196.537889:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00100000:3.0:1786177196.537894:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177196.537895:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786177196.537911:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.537923:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177196.537924:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177196.537932:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.537937:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:3.0:1786177196.537938:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177196.537942:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177196.537949:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177196.537954:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177196.537961:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177196.537969:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177196.537976:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:2.0:1786177196.537977:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177196.537984:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177196.537993:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:2.0:1786177196.537994:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177196.538003:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000008:00000001:2.0:1786177196.538004:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.538006:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177196.538009:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.538011:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000010:2.0:1786177196.538014:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000100:00000001:3.0:1786177196.538014:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:2.0:1786177196.538023:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00080000:3.0:1786177196.538023:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:2.0:1786177196.538038:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00080000:3.0:1786177196.538043:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
02000000:00000001:2.0:1786177196.538045:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.538047:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1786177196.538048:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177196.538056:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:3.0:1786177196.538063:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177196.538064:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177196.538078:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00100000:3.0:1786177196.538079:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856348672:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177196.538086:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.538088:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.538096:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177196.538100:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856348672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177196.538107:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177196.538109:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177196.538112:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177196.538117:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177196.538125:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:2.0:1786177196.538127:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000020:00000001:3.0:1786177196.538130:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00000001:2.0:1786177196.538132:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177196.538135:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000040:2.0:1786177196.538142:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177196.538142:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177196.538147:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:3.0:1786177196.538156:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000001:2.0:1786177196.538160:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177196.538161:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177196.538166:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00080000:2.0:1786177196.538168:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177196.538176:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177196.538176:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177196.538181:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177196.538182:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177196.538185:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177196.538190:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177196.538196:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177196.538197:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177196.538203:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.538206:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:2.0:1786177196.538210:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7419
00000100:00000001:3.0:1786177196.538210:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177196.538216:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177196.538226:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786177196.538230:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177196.538245:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177196.538251:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177196.538251:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177196.538259:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177196.538260:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.538270:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1786177196.538271:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1786177196.538273:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177196.538277:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177196.538279:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177196.538281:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177196.538289:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00100000:3.0:1786177196.538290:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856348800:192.168.202.110@tcp:38:kworker.0
00000020:00000040:2.0:1786177196.538296:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:3.0:1786177196.538297:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177196.538301:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:2.0:1786177196.538303:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
02000000:00000001:3.0:1786177196.538304:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177196.538308:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:2.0:1786177196.538311:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
02000000:00000010:3.0:1786177196.538312:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177196.538315:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177196.538321:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:3.0:1786177196.538322:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177196.538327:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000020:00000001:2.0:1786177196.538332:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000010:3.0:1786177196.538334:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177196.538338:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856348800, portal 10
00000100:00000001:3.0:1786177196.538342:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177196.538344:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454188800 : -114548255362816 : ffff97d1a7edf100)
00000100:00000040:2.0:1786177196.538345:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177196.538350:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177196.538352:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177196.538360:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177196.538363:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1786177196.538369:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:3.0:1786177196.538370:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177196.538375:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177196.538380:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856348800, offset 0
00000020:00000040:2.0:1786177196.538381:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000400:00000200:3.0:1786177196.538388:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786177196.538393:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1786177196.538393:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177196.538401:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1786177196.538402:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177196.538405:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:2.0:1786177196.538412:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1786177196.538412:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177196.538417:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177196.538420:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:2.0:1786177196.538421:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177196.538426:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254896:254896:256:4294967295] 192.168.202.110@tcp LPNI seq info [254885:254885:8:4294967295]
02000000:00000001:2.0:1786177196.538431:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177196.538437:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177196.538441:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177196.538448:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:3.0:1786177196.538449:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177196.538458:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000008:00000001:2.0:1786177196.538459:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1786177196.538462:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000100:00000001:2.0:1786177196.538466:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177196.538471:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177196.538475:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000100:00000010:2.0:1786177196.538479:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000800:00000200:3.1:1786177196.538481:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177196.538487:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000020:00000040:2.0:1786177196.538489:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000800:00000100:3.0:1786177196.538493:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177196.538503:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
02000000:00000001:2.0:1786177196.538505:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1786177196.538507:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1786177196.538509:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177196.538530:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177196.538533:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177196.538535:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177196.538537:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177196.538540:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177196.538545:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177196.538552:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177196.538553:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1786177196.538559:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177196.538561:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1786177196.538568:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:3.0:1786177196.538574:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177196.538578:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.538579:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177196.538590:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177196.538595:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1786177196.538599:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177196.538613:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177196.538617:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302092299
00000100:00000001:3.0:1786177196.538622:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302096078
00000100:00000001:2.0:1786177196.538623:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177196.538628:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302092299
00000100:00000001:2.0:1786177196.538629:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.538632:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302096078
00000100:00000001:3.0:1786177196.538636:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177196.538639:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177196.538642:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.538642:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177196.538651:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177196/real 1786177196]  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177196.538712:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1786177196.538714:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177196.538716:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177196.538721:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00000040:2.0:1786177196.538724:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7429)
00000400:00000200:3.0:1786177196.538727:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177196.538732:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177196.538736:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177196.538742:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177196.538751:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.538756:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177196.538757:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.538759:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177196.538760:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177196.538762:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.538779:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177196.538783:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.538789:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177196.538791:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.538792:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177196.538793:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.538794:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177196.538796:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177196.538798:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177196.538800:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177196.538801:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177196.538804:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177196.538810:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856348800:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177196.538813:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177196.538815:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.538819:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177196.538821:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177196.538822:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177196.538827:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177196.538829:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177196.538832:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856348928:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177196.538834:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177196.538836:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177196.538837:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177196.538839:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177196.538842:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177196.538843:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177196.538847:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177196.538849:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177196.538851:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177196.538853:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856348928, portal 10
00000100:00000001:3.0:1786177196.538855:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177196.538856:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000100:00000040:3.0:1786177196.538861:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.538865:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177196.538867:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177196.538868:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177196.538870:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856348928, offset 0
00000400:00000200:3.0:1786177196.538873:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177196.538876:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177196.538879:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177196.538881:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177196.538883:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177196.538885:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177196.538887:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177196.538889:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254897:254897:256:4294967295] 192.168.202.110@tcp LPNI seq info [254886:254886:8:4294967295]
00000400:00000200:3.0:1786177196.538893:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177196.538897:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177196.538900:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177196.538902:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177196.538905:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177196.538907:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786177196.538910:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177196.538912:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1786177196.538915:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177196.538916:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177196.538918:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177196.538921:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177196.538923:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177196.538926:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177196.538927:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177196.538929:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.538935:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177196.538937:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.538943:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.538945:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177196.538948:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177196.538951:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177196.538955:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177196.538956:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302092424
00000100:00000001:3.0:1786177196.538959:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302096078
00000100:00000001:3.0:1786177196.538962:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302092424
00000100:00000001:3.0:1786177196.538964:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302096078
00000100:00000001:3.0:1786177196.538967:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177196.538969:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.538971:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177196.538976:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177196/real 1786177196]  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177196.538985:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177196.538987:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177196.538989:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177196.538992:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177196.538994:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177196.538996:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177196.538998:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177196.539004:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539010:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177196.539011:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539012:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177196.539013:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177196.539015:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539020:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177196.539022:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539028:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177196.539031:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539032:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177196.539034:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539036:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177196.539038:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177196.539041:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177196.539045:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177196.539047:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177196.539051:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177196.539060:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856348928:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177196.539065:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177196.539070:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539077:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177196.539079:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177196.539082:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177196.539084:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177196.539086:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177196.539089:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856349056:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177196.539093:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177196.539095:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177196.539097:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177196.539099:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177196.539103:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177196.539105:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177196.539109:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177196.539112:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177196.539116:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177196.539119:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856349056, portal 4
00000100:00000001:3.0:1786177196.539121:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177196.539122:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178944 : -114548255372672 : ffff97d1a7edca80)
00000100:00000040:3.0:1786177196.539128:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539135:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177196.539138:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177196.539140:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177196.539143:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856349056, offset 0
00000400:00000200:3.0:1786177196.539147:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177196.539151:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177196.539155:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177196.539159:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177196.539161:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177196.539164:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177196.539166:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177196.539170:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254898:254898:256:4294967295] 192.168.202.110@tcp LPNI seq info [254887:254887:8:4294967295]
00000400:00000200:3.0:1786177196.539174:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177196.539181:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177196.539187:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177196.539190:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177196.539194:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177196.539197:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786177196.539202:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177196.539205:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1786177196.539210:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177196.539212:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177196.539215:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177196.539219:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177196.539222:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177196.539224:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177196.539227:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177196.539231:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539239:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177196.539242:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539249:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.539251:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177196.539254:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177196.539258:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177196.539262:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177196.539265:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302092299
00000100:00000001:3.0:1786177196.539268:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302096078
00000100:00000001:3.0:1786177196.539270:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302092299
00000100:00000001:3.0:1786177196.539272:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302096078
00000100:00000001:3.0:1786177196.539275:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177196.539277:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.539279:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177196.539283:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177196/real 1786177196]  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177196.539292:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177196.539293:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177196.539295:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177196.539298:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177196.539300:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177196.539302:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177196.539305:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177196.539315:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539322:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177196.539324:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539326:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177196.539328:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177196.539332:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539337:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177196.539340:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539345:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177196.539346:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539347:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177196.539348:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539349:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177196.539350:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177196.539352:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177196.539354:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177196.539355:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177196.539358:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177196.539364:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856349056:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177196.539367:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539368:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177196.539370:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf100 x1872934856348800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539374:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177196.539375:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177196.539376:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177196.539378:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177196.539380:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177196.539381:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177196.539382:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177196.539384:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:3.0:1786177196.539387:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177196.539388:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177196.539389:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf100.
00000100:00000001:3.0:1786177196.539391:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177196.539392:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539393:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177196.539396:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856348928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539400:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177196.539401:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177196.539402:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177196.539404:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177196.539405:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177196.539406:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177196.539407:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177196.539408:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:3.0:1786177196.539409:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177196.539411:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177196.539412:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
00000100:00000001:3.0:1786177196.539413:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177196.539414:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539415:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177196.539418:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edca80 x1872934856349056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539421:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177196.539422:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177196.539423:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177196.539424:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177196.539425:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177196.539425:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177196.539426:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177196.539427:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:3.0:1786177196.539428:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177196.539429:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177196.539430:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edca80.
00000100:00000001:3.0:1786177196.539431:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177196.539432:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539433:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539435:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177196.539436:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.539438:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177196.539439:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177196.539440:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177196.539442:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539446:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177196.539447:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177196.539449:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177196.539450:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177196.539451:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177196.539453:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856349184:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177196.539455:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177196.539457:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177196.539458:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177196.539460:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177196.539462:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177196.539464:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177196.539466:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177196.539468:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177196.539471:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177196.539473:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856349184, portal 4
00000100:00000001:3.0:1786177196.539474:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177196.539475:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000040:3.0:1786177196.539480:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539484:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177196.539486:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177196.539487:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000200:3.0:1786177196.539489:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856349184, offset 0
00000400:00000200:3.0:1786177196.539492:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177196.539494:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177196.539497:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177196.539499:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177196.539501:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177196.539503:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177196.539504:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177196.539506:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254899:254899:256:4294967295] 192.168.202.110@tcp LPNI seq info [254888:254888:8:4294967295]
00000400:00000200:3.0:1786177196.539509:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177196.539531:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177196.539535:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177196.539537:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000800:00000200:3.0:1786177196.539540:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177196.539542:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786177196.539545:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177196.539547:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1786177196.539550:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177196.539552:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:3.0:1786177196.539554:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177196.539557:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177196.539559:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:3.0:1786177196.539560:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:3.0:1786177196.539562:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177196.539564:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539570:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177196.539572:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539576:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.539577:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177196.539580:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177196.539582:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177196.539585:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177196.539586:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302092308
00000100:00000001:3.0:1786177196.539588:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302096078
00000100:00000001:3.0:1786177196.539590:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302092308
00000100:00000001:3.0:1786177196.539591:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302096078
00000100:00000001:3.0:1786177196.539593:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177196.539594:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.539595:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177196.539597:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177196/real 1786177196]  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177196.539602:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177196.539603:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177196.539605:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177196.539606:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177196.539607:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177196.539609:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177196.539611:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177196.539617:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539621:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177196.539621:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539623:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177196.539623:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177196.539625:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539629:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177196.539632:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539636:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177196.539637:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539639:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177196.539640:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539641:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177196.539642:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177196.539644:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177196.539646:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177196.539647:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177196.539649:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177196.539654:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856349184:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177196.539656:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539657:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177196.539659:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856349184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177196.539664:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177196.539665:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177196.539666:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177196.539668:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177196.539668:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177196.539669:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177196.539670:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177196.539672:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0800.
02000000:00000001:3.0:1786177196.539673:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177196.539674:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177196.539675:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:3.0:1786177196.539677:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177196.539678:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539679:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177196.539681:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177196.539682:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177196.539684:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177196.539685:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177197.112660:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177197.112668:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177197.112674:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177197.112675:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177197.112678:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177197.112679:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177197.112749:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177197.112753:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177197.112756:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177197.560564:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177197.560578:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177197.560585:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177197.560592:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177197.560595:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177197.560599:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177197.560603:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177197.560606:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177197.560610:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177198.136617:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177198.136624:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177198.136629:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177198.136630:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177198.136633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177198.136634:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177198.136701:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177198.136705:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177198.136708:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177198.584634:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177198.584641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177198.584643:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177198.584647:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177198.584651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177198.584654:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177198.584656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177198.584658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177198.584660:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177199.160681:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177199.160689:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177199.160694:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177199.160695:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177199.160698:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177199.160699:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177199.160772:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177199.160779:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177199.160782:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177199.608633:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177199.608646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177199.608650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177199.608658:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177199.608663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177199.608667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177199.608671:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177199.608675:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177199.608679:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177200.184652:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177200.184660:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177200.184666:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177200.184667:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177200.184670:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177200.184671:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177200.185473:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177200.185477:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177200.185480:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177200.376650:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302096078, at 4302096128
00000100:00000040:1.0:1786177200.376650:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302096078, at 4302096128
00000100:00000010:0.0:1786177200.376669:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37060.
00000100:00000010:1.0:1786177200.376674:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37ea0.
00000100:00000001:3.0:1786177200.632577:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177200.632585:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177200.632587:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177200.632592:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177200.632595:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177200.632599:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177200.632601:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177200.632604:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177200.632606:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177201.208630:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177201.208640:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177201.208642:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177201.208645:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177201.208651:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177201.208653:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177201.208655:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177201.208657:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177201.208660:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177201.208662:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177201.208663:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177201.208664:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177201.208665:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177201.208666:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177201.208667:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177201.208668:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177201.208671:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177201.208672:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177201.208732:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177201.208737:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177201.208741:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786177201.656642:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177201.656642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177201.656652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177201.656656:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177201.656657:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177201.656663:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1786177201.656665:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786177201.656668:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177201.656672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177201.656676:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177201.656678:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177201.656684:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1786177201.656688:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7424
00000100:00000001:3.0:1786177201.656689:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177201.656696:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177201.656702:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177201.656713:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177201.656717:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177201.656752:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177201.656756:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177201.656760:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177201.656762:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177201.656767:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177201.656769:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177201.656773:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177201.656777:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177201.656780:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177201.656783:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177201.656794:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177201.656797:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177201.656799:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177201.656802:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177201.656809:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177201.656813:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177201.656816:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177201.656820:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177201.656824:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177201.656827:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177201.656830:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177201.656832:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177201.656845:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786177201.656849:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177201.656854:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177201.656856:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177201.656861:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177201.656864:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177201.656869:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177201.656881:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786177201.656889:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177201.656902:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177201.656915:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177201.656921:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177201.656926:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177201.656952:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177201.656956:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177201.656959:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177201.656961:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177201.656963:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177201.656966:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177201.656969:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7424
00000400:00000200:2.0:1786177201.656974:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177201.656978:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177201.656981:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177201.656985:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177201.656988:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177201.656991:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177201.656992:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177201.656994:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177201.656998:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1786177201.656999:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177201.657003:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1786177201.657004:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177201.657008:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177201.657010:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177201.657014:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177201.657015:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177201.657018:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177201.657028:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177201.657034:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177201.657036:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1786177201.657039:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177201.657041:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177201.657046:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177201.657049:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:2.0:1786177201.657051:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
02000000:00000001:3.0:1786177201.657056:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:2.0:1786177201.657057:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1786177201.657064:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177201.657065:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177201.657070:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:3.0:1786177201.657074:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856349312:192.168.202.110@tcp:250:kworker.0
02000000:00000001:2.0:1786177201.657079:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177201.657081:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177201.657084:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177201.657088:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:3.0:1786177201.657090:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:2.0:1786177201.657094:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1786177201.657095:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1786177201.657102:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177201.657103:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786177201.657108:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177201.657113:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000010:2.0:1786177201.657116:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
02000000:00000001:3.0:1786177201.657118:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786177201.657122:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177201.657126:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1786177201.657128:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177201.657129:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177201.657134:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177201.657136:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1786177201.657137:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
02000000:00000001:2.0:1786177201.657138:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177201.657147:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000400:00000010:3.0:1786177201.657151:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:2.0:1786177201.657152:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177201.657162:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1786177201.657163:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856349312, portal 25
00000100:00000001:3.0:1786177201.657170:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177201.657174:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000001:2.0:1786177201.657177:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177201.657182:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177201.657188:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177201.657193:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177201.657208:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.657212:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1786177201.657214:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1786177201.657220:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786177201.657223:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:3.0:1786177201.657226:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e330.
00000100:00000001:2.0:1786177201.657227:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177201.657232:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1786177201.657234:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856349312, offset 0
00000100:00000001:2.0:1786177201.657237:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177201.657242:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7424
00000400:00000200:3.0:1786177201.657248:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177201.657250:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177201.657257:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177201.657260:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1786177201.657264:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786177201.657267:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177201.657271:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177201.657273:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177201.657278:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1786177201.657279:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177201.657285:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177201.657288:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177201.657290:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:3.0:1786177201.657291:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:2.0:1786177201.657294:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177201.657298:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177201.657299:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:2.0:1786177201.657303:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177201.657308:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:2.0:1786177201.657312:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177201.657317:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177201.657317:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:2.0:1786177201.657323:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177201.657328:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254900:254900:256:4294967295] 192.168.202.110@tcp LPNI seq info [254889:254889:8:4294967295]
00000100:00000001:2.0:1786177201.657329:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177201.657335:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:3.0:1786177201.657340:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786177201.657342:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177201.657349:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177201.657351:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177201.657355:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177201.657361:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177201.657362:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786177201.657366:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177201.657371:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000800:00000200:3.0:1786177201.657377:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000008:00000001:2.0:1786177201.657378:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177201.657384:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177201.657389:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000800:00000010:3.0:1786177201.657389:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cda00.
00000020:00000040:2.0:1786177201.657396:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177201.657400:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177201.657403:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177201.657404:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177201.657406:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177201.657410:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177201.657414:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177201.657415:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:2.0:1786177201.657420:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786177201.657426:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177201.657432:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177201.657436:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177201.657444:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177201.657446:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177201.657450:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.657452:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302101197
00000100:00000040:2.0:1786177201.657459:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1786177201.657463:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37d80.
00000100:00000001:2.0:1786177201.657474:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.657477:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:2.0:1786177201.657478:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177201.657485:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177201.657487:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177201.657488:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302101197
00000100:00080000:2.0:1786177201.657490:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000010:3.0:1786177201.657496:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37e40.
00000100:00000001:2.0:1786177201.657497:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177201.657504:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7424
00000100:00000001:3.0:1786177201.657504:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177201.657535:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1786177201.657538:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177201.657541:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177201.657546:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177201.657546:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177201.657553:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177201.657555:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177201.657561:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177201.657564:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177201.657567:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177201.657568:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177201.657574:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177201.657574:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1786177201.657580:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1786177201.657581:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177201.657585:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:3.0:1786177201.657586:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1786177201.657590:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177201.657591:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1786177201.657594:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177201.657600:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177201.657600:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177201.657608:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786177201.657618:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177201.657621:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177201.657623:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177201.657625:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177201.657628:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177201.657631:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1786177201.657632:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177201.657636:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1786177201.657637:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
02000000:00000001:3.0:1786177201.657643:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177201.657644:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177201.657651:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177201.657654:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856349440:192.168.202.110@tcp:38:kworker.0
02000000:00000001:2.0:1786177201.657655:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177201.657660:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177201.657662:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177201.657665:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177201.657667:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1786177201.657669:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:3.0:1786177201.657670:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1786177201.657675:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177201.657680:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177201.657682:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177201.657688:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
02000000:00000001:3.0:1786177201.657689:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:2.0:1786177201.657692:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177201.657695:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177201.657698:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177201.657700:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:3.0:1786177201.657700:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:2.0:1786177201.657704:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177201.657706:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177201.657708:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177201.657715:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000010:2.0:1786177201.657717:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000400:00000010:3.0:1786177201.657735:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580e70.
00000100:00000001:2.0:1786177201.657739:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177201.657743:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514eaa0.
00000100:00100000:2.0:1786177201.657748:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1786177201.657751:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856349440, portal 10
00000100:00000001:3.0:1786177201.657756:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177201.657760:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178944 : -114548255372672 : ffff97d1a7edca80)
00000100:00000040:3.0:1786177201.657772:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177201.657773:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177201.657779:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.657789:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177201.657790:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177201.657797:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177201.657802:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ebb0.
00000100:00000200:3.0:1786177201.657808:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856349440, offset 0
00000100:00000001:2.0:1786177201.657809:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177201.657815:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1786177201.657819:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786177201.657823:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177201.657826:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177201.657830:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1786177201.657830:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177201.657837:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177201.657840:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:2.0:1786177201.657844:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7424
00000400:00000200:3.0:1786177201.657848:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177201.657854:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177201.657856:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177201.657859:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177201.657863:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254901:254901:255:4294967295] 192.168.202.110@tcp LPNI seq info [254890:254890:7:4294967295]
00000400:00000200:3.0:1786177201.657867:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177201.657874:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177201.657881:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1786177201.657885:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177201.657891:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1786177201.657895:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177201.657900:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000010:3.0:1786177201.657903:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd700.
00000100:00000001:2.0:1786177201.657904:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177201.657910:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:3.0:1786177201.657914:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177201.657916:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177201.657921:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000200:3.1:1786177201.657924:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1786177201.657925:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177201.657929:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177201.657931:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177201.657939:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000020:00000040:2.0:1786177201.657940:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786177201.657945:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:3.0:1786177201.657946:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:2.0:1786177201.657951:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:3.0:1786177201.657952:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302097418
00000100:00000001:3.0:1786177201.657959:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302101197
00000100:00000040:2.0:1786177201.657960:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177201.657965:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302097418
00000100:00000001:2.0:1786177201.657966:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177201.657972:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177201.657973:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302101197
00000100:00000001:2.0:1786177201.657978:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177201.657980:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1786177201.657984:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:3.0:1786177201.657987:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177201.657990:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177201.657993:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786177201.657995:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177201.657999:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1786177201.658002:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177201.658004:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177201.658011:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177201.658016:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177201.658019:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177201.658022:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000008:00000001:2.0:1786177201.658024:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.658026:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177201.658028:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177201.658031:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000010:2.0:1786177201.658035:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
02000000:00000001:3.0:1786177201.658035:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177201.658042:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1786177201.658043:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177201.658050:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:3.0:1786177201.658050:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856349568:192.168.202.110@tcp:38:kworker.0
02000000:00000001:2.0:1786177201.658052:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.658058:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1786177201.658059:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177201.658062:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177201.658063:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177201.658066:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177201.658068:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177201.658071:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177201.658075:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
02000000:00000010:3.0:1786177201.658078:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:2.0:1786177201.658081:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177201.658083:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177201.658090:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1786177201.658094:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1786177201.658096:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580318.
00000400:00000010:3.0:1786177201.658103:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000001:2.0:1786177201.658110:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:3.0:1786177201.658111:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856349568, portal 10
00000100:00000001:2.0:1786177201.658116:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.658116:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177201.658121:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000100:00000040:2.0:1786177201.658125:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177201.658135:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177201.658137:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177201.658147:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7435)
00000100:00000001:3.0:1786177201.658148:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177201.658154:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177201.658158:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ed48.
00000100:00000200:3.0:1786177201.658162:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856349568, offset 0
00000400:00000200:3.0:1786177201.658168:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177201.658173:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177201.658179:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177201.658183:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177201.658185:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177201.658188:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177201.658190:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177201.658193:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254902:254902:254:4294967295] 192.168.202.110@tcp LPNI seq info [254891:254891:6:4294967295]
00000400:00000200:3.0:1786177201.658198:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177201.658205:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177201.658210:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177201.658215:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cdb00.
00000800:00000200:3.0:1786177201.658219:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177201.658223:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177201.658226:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177201.658229:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177201.658233:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177201.658235:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302097543
00000100:00000001:3.0:1786177201.658238:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302101197
00000100:00000001:3.0:1786177201.658240:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302097543
00000100:00000001:3.0:1786177201.658243:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302101197
00000100:00000001:3.0:1786177201.658246:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177201.658248:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.658251:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.658253:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177201.658257:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177201.658259:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1786177201.658262:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177201.658264:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177201.658265:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177201.658270:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177201.658278:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177201.658280:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177201.658283:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177201.658285:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177201.658287:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177201.658290:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856349696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177201.658294:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177201.658296:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177201.658298:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177201.658300:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177201.658304:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177201.658306:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177201.658310:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177201.658312:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580d68.
00000400:00000010:3.0:1786177201.658317:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e6e8.
00000100:00000200:3.0:1786177201.658320:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856349696, portal 4
00000100:00000001:3.0:1786177201.658322:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177201.658323:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454188800 : -114548255362816 : ffff97d1a7edf100)
00000100:00000040:3.0:1786177201.658330:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177201.658337:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177201.658340:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177201.658342:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ecc0.
00000100:00000200:3.0:1786177201.658345:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856349696, offset 0
00000400:00000200:3.0:1786177201.658350:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177201.658353:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177201.658357:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177201.658362:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177201.658364:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177201.658366:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177201.658369:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177201.658372:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254903:254903:253:4294967295] 192.168.202.110@tcp LPNI seq info [254892:254892:5:4294967295]
00000400:00000200:3.0:1786177201.658377:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177201.658384:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177201.658390:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177201.658394:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd800.
00000800:00000200:3.0:1786177201.658398:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177201.658403:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177201.658406:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177201.658410:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177201.658413:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177201.658416:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302097418
00000100:00000001:3.0:1786177201.658418:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302101197
00000100:00000001:3.0:1786177201.658421:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302097418
00000100:00000001:3.0:1786177201.658423:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302101197
00000100:00000001:3.0:1786177201.658425:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177201.658428:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.658430:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177201.658434:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177201.658442:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177201.658443:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177201.658445:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177201.658447:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177201.658449:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177201.658452:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856349824:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177201.658456:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177201.658458:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177201.658460:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177201.658461:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177201.658465:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177201.658467:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177201.658470:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177201.658473:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba5809f8.
00000400:00000010:3.0:1786177201.658477:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e440.
00000100:00000200:3.0:1786177201.658838:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856349824, portal 4
00000100:00000001:3.0:1786177201.658842:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177201.658843:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00000040:3.0:1786177201.658851:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177201.658858:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177201.658861:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177201.658865:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e880.
00000100:00000200:3.0:1786177201.658867:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856349824, offset 0
00000400:00000200:3.0:1786177201.658872:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177201.658877:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177201.658882:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177201.658885:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177201.658888:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177201.658891:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177201.658893:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177201.658897:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254904:254904:252:4294967295] 192.168.202.110@tcp LPNI seq info [254893:254893:4:4294967295]
00000400:00000200:3.0:1786177201.658901:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177201.658908:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177201.658913:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177201.658917:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd400.
00000800:00000200:3.0:1786177201.658921:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177201.658925:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177201.658928:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177201.658932:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177201.658935:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177201.658938:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302097427
00000100:00000001:3.0:1786177201.658940:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302101197
00000100:00000001:3.0:1786177201.658943:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302097427
00000100:00000001:3.0:1786177201.658945:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302101197
00000100:00000001:3.0:1786177201.658948:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177201.658951:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.658954:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.658957:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177201.658962:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177201.658964:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1786177201.658966:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177201.658969:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177201.658970:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177201.658973:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177202.232623:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177202.232631:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177202.232636:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177202.232637:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177202.232647:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177202.232648:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177202.232698:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177202.232702:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177202.232705:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177202.680639:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177202.680651:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177202.680654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177202.680662:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177202.680668:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177202.680673:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177202.680677:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177202.680681:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177202.680685:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177203.256633:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177203.256640:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177203.256645:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177203.256646:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177203.256648:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177203.256649:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177203.256724:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177203.256729:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177203.256732:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177203.704637:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177203.704649:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177203.704652:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177203.704659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177203.704664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177203.704669:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177203.704673:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177203.704676:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177203.704679:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177204.280662:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177204.280673:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177204.280681:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177204.280682:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177204.280686:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177204.280691:0:104859:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177204.280724:0:104859:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177204.280727:0:104859:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177204.280731:0:104859:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177204.280733:0:104859:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177204.280735:0:104859:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177204.280737:0:104859:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177204.280812:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177204.280817:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177204.280822:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177204.728649:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177204.728657:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177204.728660:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177204.728666:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177204.728671:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177204.728675:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.728678:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177204.728682:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.728685:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:0.0:1786177204.728895:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:0.0:1786177204.728950:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:0.0:1786177204.728961:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177204.728966:0:2380:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:0.0:1786177204.728972:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177204.728977:0:2380:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:0.0:1786177204.728984:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1786177204.728995:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cda00.
00000400:00000200:0.0:1786177204.729003:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1786177204.729009:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1786177204.729018:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e330
00000400:00000010:0.0:1786177204.729026:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e330.
00000100:00000001:0.0:1786177204.729034:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177204.729046:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177204.729080:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177204.729085:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177204.729094:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177204.729096:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1786177204.729108:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1786177204.729111:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd700.
00000400:00000200:0.0:1786177204.729114:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1786177204.729118:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1786177204.729121:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ebb0
00000400:00000010:0.0:1786177204.729123:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ebb0.
00000100:00000001:0.0:1786177204.729126:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177204.729132:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177204.729140:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177204.729143:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177204.729150:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177204.729151:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1786177204.729158:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1786177204.729161:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cdb00.
00000400:00000200:0.0:1786177204.729164:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1786177204.729167:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1786177204.729169:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ed48
00000400:00000010:0.0:1786177204.729171:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ed48.
00000100:00000001:0.0:1786177204.729174:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177204.729178:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177204.729187:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177204.729190:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177204.729195:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177204.729197:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1786177204.729202:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1786177204.729205:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd800.
00000400:00000200:0.0:1786177204.729208:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1786177204.729212:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1786177204.729215:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ecc0
00000400:00000010:0.0:1786177204.729216:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ecc0.
00000100:00000001:0.0:1786177204.729219:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177204.729224:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177204.729232:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177204.729235:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177204.729242:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177204.729243:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1786177204.729249:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:0.0:1786177204.729252:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd400.
00000400:00000200:0.0:1786177204.729255:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1786177204.729259:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1786177204.729261:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e880
00000400:00000010:0.0:1786177204.729264:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e880.
00000100:00000001:0.0:1786177204.729266:0:2380:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177204.729272:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177204.729281:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177204.729284:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177204.729293:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177204.729295:0:2380:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177204.730089:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177204.730111:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177204.730115:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177204.730132:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177201/real 1786177204]  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177204.730159:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177204.730162:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177204.730166:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580d68.
00000400:00000200:3.0:1786177204.730172:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e6e8
00000400:00000010:3.0:1786177204.730176:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e6e8.
00000100:00000001:3.0:1786177204.730181:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177204.730185:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177204.730197:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730205:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177204.730207:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730211:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177204.730213:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177204.730218:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730226:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177204.730232:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730241:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177204.730244:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730246:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177204.730247:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730250:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177204.730252:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177204.730256:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177204.730261:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177204.730264:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177204.730270:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177204.730279:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856349696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177204.730285:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177204.730290:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177201/real 1786177204]  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177204.730298:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177204.730299:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177204.730302:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba5809f8.
00000400:00000200:3.0:1786177204.730304:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e440
00000400:00000010:3.0:1786177204.730306:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e440.
00000100:00000001:3.0:1786177204.730309:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177204.730312:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177204.730320:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730327:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177204.730328:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730330:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177204.730331:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177204.730335:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730342:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177204.730346:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730353:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177204.730355:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730357:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177204.730358:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730360:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177204.730362:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177204.730364:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177204.730368:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177204.730370:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177204.730373:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177204.730381:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856349824:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177204.730386:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177204.730391:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177201/real 1786177204]  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177204.730399:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177204.730400:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177204.730402:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580e70.
00000400:00000200:3.0:1786177204.730405:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514eaa0
00000400:00000010:3.0:1786177204.730407:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514eaa0.
00000100:00000001:3.0:1786177204.730410:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177204.730414:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177204.730421:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730428:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177204.730430:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730431:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177204.730433:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177204.730437:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730464:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177204.730469:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730476:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177204.730478:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730480:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177204.730481:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730483:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177204.730485:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177204.730488:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177204.730491:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177204.730494:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177204.730498:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177204.730506:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856349440:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177204.730541:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177204.730547:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177201/real 1786177204]  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177204.730555:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177204.730556:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177204.730558:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580318.
00000400:00000200:3.0:1786177204.730561:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177204.730562:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177204.730564:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177204.730567:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177204.730580:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730588:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177204.730590:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730592:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177204.730595:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177204.730599:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730606:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177204.730611:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730617:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177204.730620:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730621:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177204.730622:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730624:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177204.730626:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177204.730634:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177204.730638:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177204.730640:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177204.730644:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177204.730650:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856349568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177204.730655:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177204.730659:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177201/real 1786177204]  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177204.730672:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177204.730674:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177204.730676:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177204.730679:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177204.730681:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177204.730683:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177204.730686:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177204.730707:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730713:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177204.730715:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730717:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177204.730718:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177204.730722:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730728:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177204.730732:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730739:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177204.730741:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730743:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177204.730744:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730746:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177204.730748:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177204.730751:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177204.730754:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177204.730756:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177204.730760:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177204.730768:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856349312:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177204.730772:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730775:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177204.730779:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf100 x1872934856349696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730786:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177204.730790:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177204.730793:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177204.730798:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177204.730800:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177204.730802:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177204.730805:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177204.730808:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3400.
02000000:00000001:3.0:1786177204.730811:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177204.730813:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177204.730814:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf100.
00000100:00000001:3.0:1786177204.730817:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177204.730818:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730820:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177204.730822:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856349824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730827:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177204.730828:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177204.730829:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177204.730831:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177204.730832:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177204.730832:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177204.730834:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177204.730835:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:3.0:1786177204.730836:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177204.730836:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177204.730837:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:3.0:1786177204.730839:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177204.730840:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730841:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177204.730843:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edca80 x1872934856349440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730847:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177204.730848:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177204.730849:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177204.730851:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177204.730851:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177204.730852:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177204.730853:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177204.730854:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:3.0:1786177204.730856:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177204.730856:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177204.730857:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edca80.
00000100:00000001:3.0:1786177204.730858:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177204.730859:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730860:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177204.730862:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856349568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730866:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177204.730866:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177204.730867:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177204.730869:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177204.730869:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177204.730870:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177204.730871:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177204.730872:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:3.0:1786177204.730873:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177204.730874:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177204.730875:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
00000100:00000001:3.0:1786177204.730876:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177204.730876:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730877:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177204.730879:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856349312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177204.730884:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177204.730884:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177204.730885:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177204.730887:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177204.730888:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177204.730889:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177204.730890:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177204.730891:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:3.0:1786177204.730892:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177204.730892:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177204.730893:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:3.0:1786177204.730894:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177204.730895:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730897:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177204.730903:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177204.730904:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177204.730905:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177204.730906:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177205.304620:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177205.304630:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177205.304638:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177205.304639:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177205.304643:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177205.304645:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177205.304716:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177205.304726:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177205.304730:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177205.496648:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302101197, at 4302101248
00000100:00000040:1.0:1786177205.496648:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302101197, at 4302101248
00000100:00000010:0.0:1786177205.496668:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37d80.
00000100:00000010:1.0:1786177205.496670:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37e40.
00000100:00000001:3.0:1786177205.752586:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177205.752595:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177205.752597:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177205.752603:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177205.752606:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177205.752610:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177205.752612:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177205.752615:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177205.752618:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177206.328695:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177206.328703:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177206.328708:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177206.328709:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177206.328712:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177206.328713:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177206.328781:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177206.328786:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177206.328790:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177206.776694:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177206.776705:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177206.776708:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177206.776716:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177206.776722:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177206.776727:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.776731:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177206.776735:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.776738:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177206.776949:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177206.776957:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1786177206.776960:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177206.776965:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177206.776970:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177206.776979:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7430
00000400:00000200:1.0:1786177206.776990:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177206.776998:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177206.777002:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177206.777006:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777010:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177206.777013:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777016:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177206.777021:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177206.777023:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:1.0:1786177206.777027:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177206.777031:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:1.0:1786177206.777034:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:1.0:1786177206.777037:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:1.0:1786177206.777041:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777044:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177206.777046:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777048:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177206.777051:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:1.0:1786177206.777055:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777058:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177206.777063:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177206.777069:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177206.777072:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777075:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177206.777077:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177206.777085:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa7100.
00000020:00000040:1.0:1786177206.777090:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1786177206.777095:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177206.777097:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777101:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177206.777105:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177206.777111:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177206.777122:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b00000.
00000100:00000001:1.0:1786177206.777130:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177206.777142:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177206.777153:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177206.777159:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177206.777166:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fb3d20af] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177206.777197:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177206.777202:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177206.777206:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1786177206.777207:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177206.777209:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177206.777212:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177206.777215:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7430
00000400:00000200:1.0:1786177206.777219:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177206.777222:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177206.777225:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177206.777228:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777230:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177206.777232:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777234:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177206.777236:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177206.777238:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:1.0:1786177206.777239:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177206.777242:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:1.0:1786177206.777244:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:1.0:1786177206.777246:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:1.0:1786177206.777249:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777251:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177206.777253:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777256:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177206.777258:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:1.0:1786177206.777260:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777262:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177206.777264:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177206.777273:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177206.777275:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777278:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177206.777284:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1786177206.777288:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777290:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177206.777293:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa4380.
00000020:00000040:1.0:1786177206.777296:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786177206.777298:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177206.777299:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777301:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177206.777302:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177206.777304:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177206.777308:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b02c00.
00000100:00000001:1.0:1786177206.777312:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177206.777318:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177206.777326:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177206.777328:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177206.777332:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abaf0d59] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177206.777341:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177206.777344:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177206.777348:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1786177206.777349:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177206.777350:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177206.777353:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177206.777356:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7430
00000400:00000200:1.0:1786177206.777360:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177206.777363:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177206.777365:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177206.777368:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777371:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177206.777372:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777375:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177206.777377:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177206.777379:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:1.0:1786177206.777381:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177206.777383:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:1.0:1786177206.777385:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:1.0:1786177206.777387:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:1.0:1786177206.777404:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777406:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177206.777408:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777410:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177206.777412:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:1.0:1786177206.777415:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777418:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177206.777419:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177206.777422:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177206.777424:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777426:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177206.777428:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1786177206.777430:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777432:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177206.777435:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa6d80.
00000020:00000040:1.0:1786177206.777437:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786177206.777439:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177206.777440:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777442:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177206.777444:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177206.777445:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177206.777450:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b00c00.
00000100:00000001:1.0:1786177206.777453:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177206.777458:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177206.777471:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177206.777474:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177206.777478:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038d1214c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177206.777485:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177206.777488:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177206.777492:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1786177206.777493:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177206.777495:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177206.777498:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177206.777501:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7430
00000400:00000200:1.0:1786177206.777505:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177206.777508:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177206.777546:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177206.777556:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777559:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177206.777561:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777563:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177206.777566:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177206.777567:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:1.0:1786177206.777569:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177206.777572:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:1.0:1786177206.777574:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:1.0:1786177206.777576:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:1.0:1786177206.777579:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777581:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177206.777583:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777585:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177206.777589:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:1.0:1786177206.777591:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777594:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177206.777596:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177206.777599:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177206.777601:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777603:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177206.777606:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1786177206.777609:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777611:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177206.777615:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa7b80.
00000020:00000040:1.0:1786177206.777618:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:1.0:1786177206.777621:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177206.777622:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777625:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177206.777626:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177206.777628:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177206.777632:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b03000.
00000100:00000001:1.0:1786177206.777636:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177206.777641:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177206.777649:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177206.777651:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177206.777655:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067cf9a11] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177206.777683:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177206.777686:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177206.777689:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1786177206.777690:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177206.777692:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177206.777695:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177206.777699:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7430
00000400:00000200:1.0:1786177206.777703:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177206.777706:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177206.777708:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177206.777711:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777714:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177206.777716:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777718:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177206.777721:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177206.777722:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:1.0:1786177206.777724:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177206.777728:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:1.0:1786177206.777730:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:1.0:1786177206.777732:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:1.0:1786177206.777735:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777737:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177206.777746:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777748:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177206.777751:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:1.0:1786177206.777754:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177206.777757:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177206.777759:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177206.777762:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177206.777765:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777767:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177206.777769:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1786177206.777772:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777774:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177206.777778:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa4e00.
00000020:00000040:1.0:1786177206.777780:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:1.0:1786177206.777783:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177206.777786:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177206.777787:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177206.777789:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177206.777791:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177206.777795:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b01800.
00000100:00000001:1.0:1786177206.777799:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177206.777804:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177206.777811:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177206.777813:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177206.777817:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000184a4482] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177206.777825:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177206.777832:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7440)
00000100:00000001:3.0:1786177206.778252:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177206.778259:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177206.778262:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177206.778272:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.778281:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177206.778283:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177206.778288:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177206.778290:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177206.778292:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177206.778295:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856349952:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177206.778300:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177206.778305:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177206.778307:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177206.778310:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177206.778315:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177206.778317:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177206.778322:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177206.778325:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177206.778332:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177206.778341:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856349952, portal 25
00000100:00000001:3.0:1786177206.778344:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177206.778346:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751755008 : -114547957796608 : ffff97d1b9aa7100)
00000100:00000040:3.0:1786177206.778352:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.778361:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177206.778363:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177206.778367:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177206.778372:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856349952, offset 0
00000400:00000200:3.0:1786177206.778381:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177206.778391:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177206.778400:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177206.778407:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177206.778413:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177206.778417:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177206.778421:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177206.778425:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254905:254905:256:4294967295] 192.168.202.110@tcp LPNI seq info [254894:254894:8:4294967295]
00000400:00000200:3.0:1786177206.778429:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177206.778441:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177206.778449:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177206.778456:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177206.778464:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177206.778469:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177206.778474:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177206.778477:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177206.778481:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177206.778484:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177206.778488:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177206.778492:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177206.778497:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177206.778500:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177206.778506:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177206.778534:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.778556:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177206.778560:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.778569:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177206.778572:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177206.778581:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177206.778587:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177206.778590:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302106319
00000100:00000010:3.0:1786177206.778596:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad372a0.
00000100:00000001:3.0:1786177206.778606:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177206.778614:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302106319
00000100:00000010:3.0:1786177206.778618:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37a80.
00000100:00000001:3.0:1786177206.778621:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177206.778625:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177206.778628:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177206.778631:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177206.778637:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177206/real 1786177206]  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177206.778647:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177206.778650:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177206.778652:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177206.778655:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177206.778657:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177206.778660:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177206.778664:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177206.778686:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.778693:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177206.778696:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.778698:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177206.778701:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177206.778706:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.778713:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177206.778718:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.778725:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177206.778728:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.778730:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177206.778731:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.778735:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177206.778738:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177206.778754:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177206.778758:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177206.778761:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177206.778765:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177206.778773:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856349952:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177206.778778:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177206.778781:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.778787:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177206.778789:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177206.778791:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177206.778793:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177206.778795:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177206.778798:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177206.778801:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177206.778803:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177206.778805:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177206.778807:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177206.778811:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177206.778813:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177206.778817:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177206.778820:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177206.778824:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177206.778827:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856350080, portal 10
00000100:00000001:3.0:1786177206.778829:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177206.778830:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751743360 : -114547957808256 : ffff97d1b9aa4380)
00000100:00000040:3.0:1786177206.778837:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.778845:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177206.778847:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177206.778850:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177206.778852:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856350080, offset 0
00000400:00000200:3.0:1786177206.778857:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177206.778862:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177206.778866:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177206.778869:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177206.778871:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177206.778874:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177206.778876:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177206.778879:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254906:254906:256:4294967295] 192.168.202.110@tcp LPNI seq info [254895:254895:8:4294967295]
00000400:00000200:3.0:1786177206.778885:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177206.778891:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177206.778896:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177206.778900:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177206.778905:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177206.778908:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177206.778913:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177206.778916:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177206.778920:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177206.778922:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177206.778925:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177206.778929:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177206.778933:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177206.778935:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177206.778938:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177206.778942:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.778951:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177206.778954:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.778968:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177206.778969:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177206.778974:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177206.778978:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177206.778982:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177206.778984:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302102539
00000100:00000001:3.0:1786177206.778988:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302106319
00000100:00000001:3.0:1786177206.778990:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302102539
00000100:00000001:3.0:1786177206.778992:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302106319
00000100:00000001:3.0:1786177206.778994:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177206.778996:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177206.778999:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177206.779002:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177206/real 1786177206]  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177206.779010:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177206.779012:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177206.779014:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177206.779017:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177206.779019:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177206.779021:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177206.779024:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177206.779034:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779040:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177206.779042:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.779044:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177206.779045:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177206.779049:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779057:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177206.779061:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779069:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177206.779072:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.779074:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177206.779075:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.779077:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177206.779079:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177206.779082:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177206.779086:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177206.779089:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177206.779095:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177206.779103:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177206.779107:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177206.779111:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779130:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177206.779132:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177206.779135:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177206.779141:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177206.779143:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177206.779147:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350208:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177206.779151:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177206.779154:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177206.779156:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177206.779158:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177206.779162:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177206.779164:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177206.779168:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177206.779171:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177206.779176:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177206.779179:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856350208, portal 10
00000100:00000001:3.0:1786177206.779181:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177206.779183:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751754112 : -114547957797504 : ffff97d1b9aa6d80)
00000100:00000040:3.0:1786177206.779189:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779196:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177206.779199:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177206.779201:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177206.779204:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856350208, offset 0
00000400:00000200:3.0:1786177206.779209:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177206.779214:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177206.779218:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177206.779222:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177206.779224:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177206.779226:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177206.779228:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177206.779231:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254907:254907:256:4294967295] 192.168.202.110@tcp LPNI seq info [254896:254896:8:4294967295]
00000400:00000200:3.0:1786177206.779236:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177206.779243:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177206.779248:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177206.779251:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177206.779256:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177206.779258:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177206.779263:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177206.779266:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177206.779270:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177206.779273:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177206.779276:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177206.779280:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177206.779284:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177206.779286:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177206.779289:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177206.779293:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779302:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177206.779306:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779313:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177206.779315:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177206.779319:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177206.779323:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177206.779327:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177206.779329:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302102664
00000100:00000001:3.0:1786177206.779332:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302106319
00000100:00000001:3.0:1786177206.779334:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302102664
00000100:00000001:3.0:1786177206.779336:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302106319
00000100:00000001:3.0:1786177206.779339:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177206.779341:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177206.779343:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177206.779348:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177206/real 1786177206]  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177206.779356:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177206.779358:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177206.779359:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177206.779362:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177206.779364:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177206.779368:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177206.779372:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177206.779380:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779386:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177206.779388:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.779389:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177206.779391:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177206.779394:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779400:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177206.779404:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779411:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177206.779413:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.779415:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177206.779416:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.779418:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177206.779420:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177206.779422:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177206.779426:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177206.779428:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177206.779432:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177206.779441:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350208:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177206.779445:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177206.779449:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779456:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177206.779458:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177206.779461:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177206.779462:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177206.779464:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177206.779467:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350336:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177206.779471:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177206.779473:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177206.779475:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177206.779477:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177206.779481:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177206.779482:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177206.779485:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177206.779487:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177206.779491:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177206.779495:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856350336, portal 4
00000100:00000001:3.0:1786177206.779497:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177206.779498:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751757696 : -114547957793920 : ffff97d1b9aa7b80)
00000100:00000040:3.0:1786177206.779505:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779552:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177206.779555:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177206.779558:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177206.779561:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856350336, offset 0
00000400:00000200:3.0:1786177206.779566:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177206.779569:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177206.779572:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177206.779574:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177206.779576:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177206.779578:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177206.779580:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177206.779583:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254908:254908:256:4294967295] 192.168.202.110@tcp LPNI seq info [254897:254897:8:4294967295]
00000400:00000200:3.0:1786177206.779586:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177206.779594:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177206.779598:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177206.779602:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177206.779606:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177206.779610:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177206.779615:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177206.779618:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177206.779622:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177206.779624:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177206.779627:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177206.779630:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177206.779633:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177206.779635:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177206.779638:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177206.779642:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779656:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177206.779660:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779666:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177206.779668:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177206.779683:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177206.779687:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177206.779690:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177206.779693:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302102540
00000100:00000001:3.0:1786177206.779695:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302106319
00000100:00000001:3.0:1786177206.779698:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302102540
00000100:00000001:3.0:1786177206.779700:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302106319
00000100:00000001:3.0:1786177206.779703:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177206.779705:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177206.779707:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177206.779712:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177206/real 1786177206]  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177206.779721:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177206.779722:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177206.779724:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177206.779727:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177206.779729:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177206.779731:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177206.779735:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177206.779744:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779750:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177206.779752:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.779754:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177206.779756:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177206.779759:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779766:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177206.779770:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779777:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177206.779779:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.779781:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177206.779782:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.779784:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177206.779786:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177206.779789:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177206.779793:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177206.779795:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177206.779800:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177206.779808:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350336:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177206.779812:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177206.779816:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779822:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177206.779824:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177206.779826:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177206.779828:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177206.779830:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177206.779833:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177206.779836:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177206.779838:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177206.779840:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177206.779842:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177206.779846:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177206.779848:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177206.779851:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177206.779854:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177206.779856:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177206.779859:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856350464, portal 4
00000100:00000001:3.0:1786177206.779861:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177206.779863:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751746048 : -114547957805568 : ffff97d1b9aa4e00)
00000100:00000040:3.0:1786177206.779868:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779874:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177206.779877:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177206.779879:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177206.779881:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856350464, offset 0
00000400:00000200:3.0:1786177206.779886:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177206.779889:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177206.779893:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177206.779896:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177206.779898:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177206.779901:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177206.779904:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177206.779907:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254909:254909:256:4294967295] 192.168.202.110@tcp LPNI seq info [254898:254898:8:4294967295]
00000400:00000200:3.0:1786177206.779911:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177206.779917:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177206.779922:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177206.779925:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177206.779929:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177206.779932:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177206.779935:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177206.779938:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177206.779942:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177206.779944:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177206.779947:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177206.779950:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177206.779953:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177206.779955:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177206.779958:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177206.779961:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779969:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177206.779972:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.779979:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177206.779980:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177206.779984:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177206.779987:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177206.779991:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177206.779993:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302102548
00000100:00000001:3.0:1786177206.779995:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302106319
00000100:00000001:3.0:1786177206.779997:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302102548
00000100:00000001:3.0:1786177206.780000:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302106319
00000100:00000001:3.0:1786177206.780002:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177206.780004:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177206.780006:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177206.780010:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177206/real 1786177206]  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177206.780018:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177206.780020:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177206.780022:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177206.780024:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177206.780026:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177206.780028:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177206.780031:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177206.780040:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.780048:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177206.780050:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.780052:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177206.780053:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177206.780057:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.780064:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177206.780068:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.780075:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177206.780077:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.780079:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177206.780081:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.780082:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177206.780084:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177206.780087:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177206.780091:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177206.780093:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177206.780097:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177206.780106:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177206.780110:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.780113:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177206.780117:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa7100 x1872934856349952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.780125:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177206.780128:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177206.780131:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177206.780134:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177206.780137:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177206.780139:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177206.780142:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177206.780145:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00000.
02000000:00000001:3.0:1786177206.780149:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177206.780151:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177206.780154:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa7100.
00000100:00000001:3.0:1786177206.780158:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177206.780159:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.780162:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177206.780166:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4380 x1872934856350080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.780173:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177206.780174:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177206.780176:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177206.780178:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177206.780180:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177206.780181:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177206.780183:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177206.780185:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b02c00.
02000000:00000001:3.0:1786177206.780188:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177206.780189:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177206.780190:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4380.
00000100:00000001:3.0:1786177206.780193:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177206.780194:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.780196:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177206.780200:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa6d80 x1872934856350208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.780208:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177206.780209:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177206.780210:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177206.780214:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177206.780215:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177206.780217:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177206.780218:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177206.780219:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00c00.
02000000:00000001:3.0:1786177206.780221:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177206.780222:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177206.780223:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa6d80.
00000100:00000001:3.0:1786177206.780225:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177206.780226:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.780227:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177206.780230:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa7b80 x1872934856350336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.780243:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177206.780245:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177206.780246:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177206.780249:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177206.780251:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177206.780252:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177206.780254:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177206.780256:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03000.
02000000:00000001:3.0:1786177206.780258:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177206.780259:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177206.780261:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa7b80.
00000100:00000001:3.0:1786177206.780263:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177206.780265:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.780266:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177206.780271:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4e00 x1872934856350464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177206.780278:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177206.780280:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177206.780281:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177206.780284:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177206.780285:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177206.780287:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177206.780288:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177206.780289:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b01800.
02000000:00000001:3.0:1786177206.780293:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177206.780293:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177206.780295:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4e00.
00000100:00000001:3.0:1786177206.780296:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177206.780297:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.780299:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177206.780308:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177206.780309:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177206.780311:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177206.780313:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177207.352580:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177207.352588:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177207.352594:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177207.352595:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177207.352598:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177207.352599:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177207.352861:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177207.352865:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177207.352868:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177207.800600:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177207.800606:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177207.800608:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177207.800612:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177207.800615:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177207.800618:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177207.800620:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177207.800622:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177207.800624:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177208.376614:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177208.376620:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177208.376624:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177208.376625:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177208.376627:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177208.376628:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177208.376723:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177208.376728:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177208.376732:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177208.824619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177208.824625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177208.824627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177208.824630:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177208.824632:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177208.824635:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177208.824637:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177208.824639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177208.824641:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177209.400681:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177209.400689:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177209.400694:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177209.400695:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177209.400698:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177209.400699:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177209.400771:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177209.400776:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177209.400780:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177209.848694:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177209.848705:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177209.848708:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177209.848715:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177209.848719:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177209.848725:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177209.848728:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177209.848732:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177209.848735:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177210.424659:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177210.424667:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177210.424672:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177210.424673:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177210.424675:0:104859:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177210.424676:0:104859:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177210.424750:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177210.424754:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177210.424757:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177210.616665:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302106319, at 4302106368
00000100:00000040:1.0:1786177210.616665:0:104859:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302106319, at 4302106368
00000100:00000010:0.0:1786177210.616684:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad372a0.
00000100:00000010:1.0:1786177210.616687:0:104859:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37a80.
00000100:00000001:3.0:1786177210.872672:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177210.872682:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177210.872685:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177210.872693:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177210.872697:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177210.872700:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177210.872703:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177210.872706:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177210.872708:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177211.448620:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177211.448637:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177211.448639:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177211.448642:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177211.448647:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177211.448649:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177211.448650:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177211.448652:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177211.448656:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177211.448657:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177211.448659:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177211.448660:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177211.448662:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177211.448663:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177211.448664:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177211.448666:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177211.448670:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177211.448670:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177211.448757:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177211.448763:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177211.448767:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177211.896614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177211.896642:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177211.896644:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177211.896648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177211.896651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177211.896655:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177211.896657:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177211.896659:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177211.896661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177211.896679:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177211.896683:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177211.896685:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177211.896687:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177211.896691:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177211.896698:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7435
00000400:00000200:3.0:1786177211.896707:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177211.896714:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177211.896717:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177211.896721:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.896724:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177211.896728:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.896731:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177211.896735:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177211.896737:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177211.896740:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177211.896744:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177211.896746:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177211.896749:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177211.896752:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.896754:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177211.896756:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.896759:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177211.896762:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177211.896766:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177211.896769:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177211.896772:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177211.896775:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177211.896779:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.896781:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177211.896783:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177211.896792:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d180.
00000020:00000040:3.0:1786177211.896795:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177211.896798:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177211.896800:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.896802:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177211.896805:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177211.896809:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177211.896815:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:3.0:1786177211.896821:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177211.896828:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177211.896837:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177211.896840:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177211.896844:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085df314e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177211.896870:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177211.896872:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177211.896875:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177211.896876:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177211.896877:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177211.896880:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177211.896881:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7435
00000400:00000200:3.0:1786177211.896885:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177211.896888:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177211.896890:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177211.896892:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.896895:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177211.896897:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.896899:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177211.896901:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177211.896902:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177211.896903:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177211.896906:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786177211.896908:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786177211.896910:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177211.896913:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.896914:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177211.896915:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.896917:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177211.896918:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:0.0:1786177211.896921:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:3.0:1786177211.896921:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177211.896930:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177211.896932:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177211.896933:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
02000000:00000001:3.0:1786177211.896937:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177211.896939:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:3.0:1786177211.896940:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.896944:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177211.896947:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177211.896949:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.896952:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177211.896960:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1786177211.896960:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e300.
00000020:00000040:3.0:1786177211.896968:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177211.896978:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177211.896981:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.896991:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177211.896995:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177211.897001:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177211.897002:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177211.897006:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177211.897007:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786177211.897013:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177211.897020:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177211.897022:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177211.897025:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177211.897033:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856350592:192.168.202.110@tcp:250:kworker.0
00000100:00100000:3.0:1786177211.897034:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177211.897043:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177211.897049:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177211.897052:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177211.897056:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1786177211.897063:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000010:0.0:1786177211.897069:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6800.
00000100:00000001:3.0:1786177211.897069:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177211.897074:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177211.897082:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e3752a60] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177211.897083:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177211.897089:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177211.897102:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000001:3.0:1786177211.897104:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177211.897108:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177211.897111:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177211.897112:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:0.0:1786177211.897116:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856350592, portal 25
00000100:00080000:3.0:1786177211.897117:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177211.897130:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177211.897130:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786177211.897136:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297088 : -114547945254528 : ffff97d1ba69d180)
00000100:00080000:3.0:1786177211.897137:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7435
00000400:00000200:3.0:1786177211.897149:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177211.897156:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:0.0:1786177211.897158:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786177211.897159:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177211.897169:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.897178:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177211.897182:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.897189:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177211.897194:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:0.0:1786177211.897195:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:3.0:1786177211.897200:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177211.897202:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177211.897207:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000100:00000040:0.0:1786177211.897211:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000020:00000001:3.0:1786177211.897212:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000010:0.0:1786177211.897220:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000020:00000001:3.0:1786177211.897225:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000200:0.0:1786177211.897230:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856350592, offset 0
00000100:00000040:3.0:1786177211.897231:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.897237:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177211.897242:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.897244:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177211.897246:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:0.0:1786177211.897254:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:3.0:1786177211.897255:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177211.897262:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177211.897263:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177211.897267:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177211.897269:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.897270:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177211.897273:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1786177211.897274:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:3.0:1786177211.897281:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.897284:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177211.897289:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000010:3.0:1786177211.897290:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c700.
00000020:00000040:3.0:1786177211.897304:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:0.0:1786177211.897307:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1786177211.897310:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177211.897311:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.897313:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1786177211.897315:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:3.0:1786177211.897317:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1786177211.897321:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1786177211.897321:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177211.897332:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000010:3.0:1786177211.897333:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000400:00000200:0.0:1786177211.897343:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254910:254910:256:4294967295] 192.168.202.110@tcp LPNI seq info [254899:254899:8:4294967295]
00000100:00000001:3.0:1786177211.897343:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177211.897360:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1786177211.897361:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177211.897373:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177211.897375:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177211.897381:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e56458ae] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177211.897393:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177211.897395:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177211.897397:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177211.897398:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177211.897399:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177211.897404:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177211.897408:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7435
00000400:00000200:0.0:1786177211.897413:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1786177211.897418:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177211.897426:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177211.897429:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177211.897433:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:0.0:1786177211.897437:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177211.897437:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177211.897447:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000010:0.0:1786177211.897450:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000100:00000001:3.0:1786177211.897450:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177211.897457:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177211.897459:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177211.897463:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:0.0:1786177211.897465:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:3.0:1786177211.897465:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177211.897476:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000200:0.0:1786177211.897478:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:3.0:1786177211.897479:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177211.897496:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:0.1:1786177211.897497:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177211.897503:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:0.1:1786177211.897505:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:3.0:1786177211.897538:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177211.897543:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177211.897547:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177211.897555:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177211.897558:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177211.897559:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177211.897562:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000100:0.0:1786177211.897565:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786177211.897569:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1786177211.897571:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000100:00000001:3.0:1786177211.897573:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177211.897580:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:0.0:1786177211.897583:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00000001:3.0:1786177211.897584:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.897596:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177211.897602:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000010:3.0:1786177211.897605:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c000.
00000400:00000200:0.0:1786177211.897609:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000020:00000040:3.0:1786177211.897610:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000400:00000010:0.0:1786177211.897615:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
02000000:00000001:3.0:1786177211.897615:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177211.897632:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.897636:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177211.897639:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1786177211.897640:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:3.0:1786177211.897642:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786177211.897647:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1786177211.897649:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786177211.897660:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177211.897675:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177211.897680:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786177211.897682:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177211.897684:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177211.897687:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006fe8fdb7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177211.897700:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177211.897705:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177211.897714:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177211.897724:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177211.897726:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177211.897727:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177211.897732:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177211.897736:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177211.897738:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.897743:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1786177211.897747:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177211.897752:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302111438
00000100:00080000:3.0:1786177211.897752:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7435
00000100:00000010:0.0:1786177211.897760:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000400:00000200:3.0:1786177211.897768:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177211.897770:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000400:00000200:3.0:1786177211.897776:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1786177211.897779:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302111438
00000100:00080000:3.0:1786177211.897781:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000010:0.0:1786177211.897783:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134ba0.
00000100:00000001:0.0:1786177211.897792:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:3.0:1786177211.897793:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177211.897801:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177211.897802:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177211.897808:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177211.897810:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177211.897816:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1786177211.897817:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177211.897822:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00100000:0.0:1786177211.897826:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177211/real 1786177211]  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177211.897826:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177211.897835:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177211.897841:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000400:00000001:0.0:1786177211.897849:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:3.0:1786177211.897850:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000001:0.0:1786177211.897854:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177211.897862:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000020:00000001:3.0:1786177211.897862:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:0.0:1786177211.897873:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000100:00000040:3.0:1786177211.897876:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:0.0:1786177211.897878:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:3.0:1786177211.897882:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177211.897884:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:3.0:1786177211.897887:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:0.0:1786177211.897893:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177211.897894:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177211.897902:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786177211.897913:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177211.897922:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177211.897925:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:0.0:1786177211.897929:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177211.897930:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177211.897936:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.897943:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177211.897946:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177211.897957:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.897963:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177211.897975:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ea00.
00000100:00000001:0.0:1786177211.897977:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177211.897984:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:3.0:1786177211.897984:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:0.0:1786177211.897999:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1786177211.897999:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1786177211.898009:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177211.898009:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177211.898026:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177211.898029:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:0.0:1786177211.898031:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177211.898033:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177211.898045:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:0.0:1786177211.898047:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177211.898056:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177211.898057:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898070:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177211.898075:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177211.898076:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177211.898077:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177211.898080:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898087:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177211.898090:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177211.898100:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:3.0:1786177211.898109:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00080000:0.0:1786177211.898114:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177211.898115:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177211.898121:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177211.898128:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053162887] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177211.898134:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177211.898185:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177211.898188:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856350592:192.168.202.110@tcp:250:kworker.0
00000100:00000040:3.0:1786177211.898192:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7445)
00000100:00000001:0.0:1786177211.898194:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898197:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177211.898200:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d180 x1872934856350592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898207:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177211.898210:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177211.898213:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177211.898217:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177211.898219:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177211.898222:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177211.898223:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177211.898226:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786177211.898230:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177211.898232:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177211.898234:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d180.
00000100:00000001:0.0:1786177211.898237:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177211.898238:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898240:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898247:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177211.898248:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177211.898250:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177211.898252:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177211.898253:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177211.898256:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898262:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177211.898263:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177211.898265:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177211.898267:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177211.898269:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177211.898271:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177211.898273:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177211.898275:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177211.898277:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177211.898279:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177211.898281:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177211.898283:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177211.898286:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177211.898289:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177211.898291:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177211.898294:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856350720, portal 10
00000100:00000001:0.0:1786177211.898295:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177211.898297:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:0.0:1786177211.898302:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898307:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177211.898309:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177211.898311:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177211.898312:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856350720, offset 0
00000400:00000200:0.0:1786177211.898315:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177211.898319:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177211.898322:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177211.898324:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177211.898326:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177211.898328:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177211.898329:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177211.898332:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254911:254911:256:4294967295] 192.168.202.110@tcp LPNI seq info [254900:254900:8:4294967295]
00000400:00000200:0.0:1786177211.898335:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177211.898340:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177211.898343:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177211.898346:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177211.898349:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177211.898351:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786177211.898355:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177211.898356:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786177211.898359:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177211.898361:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177211.898363:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177211.898365:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177211.898368:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177211.898369:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177211.898371:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177211.898373:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898378:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177211.898380:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898390:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177211.898391:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177211.898393:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177211.898396:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177211.898398:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177211.898400:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302107658
00000100:00000001:0.0:1786177211.898402:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302111438
00000100:00000001:0.0:1786177211.898404:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302107658
00000100:00000001:0.0:1786177211.898405:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302111438
00000100:00000001:0.0:1786177211.898406:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177211.898408:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177211.898409:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177211.898411:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177211/real 1786177211]  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177211.898416:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177211.898417:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177211.898418:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177211.898420:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177211.898421:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177211.898422:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177211.898424:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177211.898429:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898433:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177211.898434:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898435:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177211.898436:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177211.898438:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898444:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177211.898446:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898451:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177211.898453:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898453:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177211.898454:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898455:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177211.898457:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177211.898458:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177211.898460:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177211.898462:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177211.898464:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177211.898469:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177211.898471:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177211.898474:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898478:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177211.898479:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177211.898481:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177211.898482:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177211.898484:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177211.898486:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177211.898488:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177211.898489:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177211.898490:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177211.898492:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177211.898494:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177211.898495:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177211.898497:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177211.898499:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177211.898502:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177211.898505:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856350848, portal 10
00000100:00000001:0.0:1786177211.898507:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177211.898509:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:0.0:1786177211.898540:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898548:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177211.898550:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177211.898553:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177211.898557:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856350848, offset 0
00000400:00000200:0.0:1786177211.898561:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177211.898566:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177211.898570:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177211.898573:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177211.898575:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177211.898577:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177211.898579:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177211.898582:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254912:254912:256:4294967295] 192.168.202.110@tcp LPNI seq info [254901:254901:8:4294967295]
00000400:00000200:0.0:1786177211.898585:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177211.898589:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177211.898593:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177211.898595:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177211.898598:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177211.898601:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786177211.898603:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177211.898605:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786177211.898608:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177211.898610:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177211.898612:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177211.898615:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177211.898617:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177211.898618:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177211.898635:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177211.898638:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898644:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177211.898646:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898651:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177211.898652:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177211.898655:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177211.898657:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177211.898659:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177211.898661:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302107784
00000100:00000001:0.0:1786177211.898663:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302111438
00000100:00000001:0.0:1786177211.898664:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302107784
00000100:00000001:0.0:1786177211.898666:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302111438
00000100:00000001:0.0:1786177211.898667:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177211.898669:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177211.898671:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177211.898675:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177211/real 1786177211]  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177211.898683:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177211.898684:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177211.898686:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177211.898689:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177211.898691:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177211.898694:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177211.898698:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177211.898707:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898715:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177211.898716:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898718:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177211.898720:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177211.898723:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898728:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177211.898731:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898735:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177211.898737:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898738:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177211.898738:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898740:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177211.898741:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177211.898743:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177211.898745:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177211.898746:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177211.898748:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177211.898753:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177211.898756:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177211.898759:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898764:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177211.898765:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177211.898767:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177211.898768:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177211.898769:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177211.898771:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177211.898773:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177211.898775:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177211.898776:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177211.898778:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177211.898781:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177211.898782:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177211.898785:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177211.898787:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177211.898789:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177211.898792:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856350976, portal 4
00000100:00000001:0.0:1786177211.898793:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177211.898794:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:0.0:1786177211.898799:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898804:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177211.898806:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177211.898807:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177211.898809:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856350976, offset 0
00000400:00000200:0.0:1786177211.898811:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177211.898814:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177211.898816:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177211.898818:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177211.898820:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177211.898822:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177211.898824:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177211.898827:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254913:254913:256:4294967295] 192.168.202.110@tcp LPNI seq info [254902:254902:8:4294967295]
00000400:00000200:0.0:1786177211.898830:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177211.898834:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177211.898837:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177211.898839:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177211.898842:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177211.898844:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786177211.898847:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177211.898849:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786177211.898852:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177211.898853:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177211.898855:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177211.898858:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177211.898860:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177211.898861:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177211.898863:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177211.898865:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898870:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177211.898872:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898877:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177211.898878:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177211.898880:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177211.898882:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177211.898884:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177211.898886:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302107659
00000100:00000001:0.0:1786177211.898887:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302111438
00000100:00000001:0.0:1786177211.898889:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302107659
00000100:00000001:0.0:1786177211.898891:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302111438
00000100:00000001:0.0:1786177211.898892:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177211.898894:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177211.898895:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177211.898898:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177211/real 1786177211]  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177211.898903:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177211.898904:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177211.898905:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177211.898906:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177211.898907:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177211.898908:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177211.898911:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177211.898920:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898927:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177211.898929:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898931:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177211.898933:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177211.898936:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898944:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177211.898949:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898957:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177211.898960:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898961:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177211.898963:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.898965:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177211.898967:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177211.898969:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177211.898972:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177211.898974:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177211.898977:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177211.898983:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856350976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177211.898986:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177211.898988:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.898992:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177211.898993:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177211.898995:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177211.898996:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177211.898997:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177211.898999:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856351104:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177211.899002:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177211.899003:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177211.899005:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177211.899006:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177211.899009:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786177211.899010:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177211.899012:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177211.899014:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177211.899016:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177211.899018:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856351104, portal 4
00000100:00000001:0.0:1786177211.899020:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177211.899021:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:0.0:1786177211.899025:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.899030:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177211.899033:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177211.899036:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177211.899038:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856351104, offset 0
00000400:00000200:0.0:1786177211.899043:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177211.899047:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177211.899051:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177211.899054:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177211.899056:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177211.899059:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177211.899061:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177211.899065:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254914:254914:256:4294967295] 192.168.202.110@tcp LPNI seq info [254903:254903:8:4294967295]
00000400:00000200:0.0:1786177211.899070:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177211.899077:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177211.899082:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177211.899085:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177211.899089:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177211.899092:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1786177211.899096:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177211.899099:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:0.0:1786177211.899103:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177211.899105:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177211.899109:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177211.899113:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177211.899116:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177211.899118:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177211.899121:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177211.899124:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.899133:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177211.899137:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.899142:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177211.899144:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177211.899147:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177211.899150:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177211.899152:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177211.899154:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302107667
00000100:00000001:0.0:1786177211.899156:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302111438
00000100:00000001:0.0:1786177211.899157:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302107667
00000100:00000001:0.0:1786177211.899159:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302111438
00000100:00000001:0.0:1786177211.899161:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177211.899162:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177211.899163:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177211.899166:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177211/real 1786177211]  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177211.899171:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177211.899172:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177211.899174:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177211.899175:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177211.899176:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177211.899178:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177211.899180:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177211.899186:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.899189:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177211.899190:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.899191:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177211.899192:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177211.899194:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.899198:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177211.899200:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.899207:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177211.899210:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.899211:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177211.899213:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.899215:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177211.899217:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177211.899219:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177211.899223:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177211.899226:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177211.899230:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177211.899239:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856351104:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177211.899244:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.899245:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177211.899249:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856350720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.899256:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177211.899258:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177211.899260:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177211.899263:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177211.899265:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177211.899266:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177211.899269:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177211.899271:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786177211.899275:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177211.899276:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177211.899278:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:0.0:1786177211.899281:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177211.899283:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.899285:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177211.899289:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856350848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.899297:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177211.899299:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177211.899300:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177211.899303:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177211.899304:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177211.899305:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177211.899308:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177211.899310:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786177211.899313:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177211.899314:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177211.899315:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:0.0:1786177211.899318:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177211.899319:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.899321:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177211.899325:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856350976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.899333:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177211.899334:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177211.899335:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177211.899338:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177211.899339:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177211.899340:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177211.899343:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177211.899345:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786177211.899347:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177211.899348:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177211.899349:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:0.0:1786177211.899352:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177211.899353:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.899355:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177211.899359:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856351104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177211.899369:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177211.899370:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177211.899371:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786177211.899373:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177211.899374:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177211.899376:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177211.899377:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177211.899379:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177211.899382:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177211.899383:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177211.899384:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:0.0:1786177211.899386:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177211.899388:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.899390:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177211.899394:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177211.899395:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177211.899397:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177211.899399:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177212.472634:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177212.472640:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177212.472644:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177212.472645:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177212.472647:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177212.472648:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177212.472738:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177212.472746:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177212.472751:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177212.920677:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177212.920684:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177212.920686:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177212.920691:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177212.920694:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177212.920697:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177212.920699:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177212.920701:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177212.920703:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177213.496642:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177213.496650:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177213.496655:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177213.496656:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177213.496659:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177213.496660:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177213.496756:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177213.496764:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177213.496769:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177213.944648:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177213.944659:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177213.944663:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177213.944671:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177213.944685:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177213.944690:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177213.944693:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177213.944697:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177213.944701:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177214.520703:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177214.520714:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177214.520723:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177214.520725:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177214.520729:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177214.520734:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177214.520736:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177214.520741:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177214.520744:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177214.520747:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177214.520750:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177214.520753:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177214.520818:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177214.520822:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177214.520825:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177214.968688:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177214.968696:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177214.968698:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177214.968704:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177214.968708:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177214.968711:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177214.968714:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177214.968717:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177214.968720:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177215.544640:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177215.544647:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177215.544651:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177215.544652:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177215.544655:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177215.544656:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177215.544720:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177215.544727:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177215.544732:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177215.736680:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302111438, at 4302111488
00000100:00000040:1.0:1786177215.736682:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302111438, at 4302111488
00000100:00000010:1.0:1786177215.736695:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134ba0.
00000100:00000010:0.0:1786177215.736697:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:0.0:1786177215.992575:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177215.992606:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177215.992608:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177215.992614:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177215.992617:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177215.992620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177215.992622:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177215.992624:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177215.992626:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177216.568635:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177216.568641:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177216.568646:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177216.568647:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177216.568650:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177216.568651:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177216.568708:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177216.568712:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177216.568716:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177217.016618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1786177217.016618:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177217.016626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177217.016629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177217.016630:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177217.016634:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:0.0:1786177217.016636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:3.0:1786177217.016638:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177217.016642:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786177217.016646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1786177217.016648:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7440
00000100:00000001:0.0:1786177217.016650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.016654:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177217.016656:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1786177217.016660:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177217.016661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177217.016668:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177217.016672:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177217.016674:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.016677:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177217.016679:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.016680:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177217.016684:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177217.016686:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177217.016689:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177217.016691:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177217.016693:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177217.016695:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177217.016697:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.016698:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177217.016700:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.016701:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177217.016703:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177217.016705:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177217.016707:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177217.016709:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177217.016711:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177217.016713:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177217.016715:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177217.016716:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177217.016723:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d500.
00000020:00000040:3.0:1786177217.016726:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177217.016729:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177217.016730:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177217.016733:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177217.016735:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177217.016739:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177217.016745:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:3.0:1786177217.016751:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177217.016758:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177217.016766:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177217.016769:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177217.016773:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7c36393] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177217.016790:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177217.016793:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177217.016795:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177217.016796:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177217.016798:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177217.016799:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177217.016801:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7440
00000400:00000200:3.0:1786177217.016804:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177217.016806:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177217.016807:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177217.016810:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.016812:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177217.016813:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.016815:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177217.016817:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177217.016817:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177217.016818:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177217.016820:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:0.0:1786177217.016821:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1786177217.016821:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786177217.016826:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786177217.016827:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1786177217.016828:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177217.016832:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1786177217.016832:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177217.016833:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.016837:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177217.016839:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000040:0.0:1786177217.016844:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786177217.016844:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177217.016849:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177217.016853:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177217.016857:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177217.016859:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1786177217.016861:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.016862:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177217.016864:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:0.0:1786177217.016867:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:3.0:1786177217.016868:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1786177217.016870:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:3.0:1786177217.016872:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177217.016875:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177217.016876:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177217.016883:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f100.
00000100:00100000:0.0:1786177217.016884:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856351232:192.168.202.110@tcp:250:kworker.0
00000020:00000040:3.0:1786177217.016886:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:0.0:1786177217.016892:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177217.016892:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177217.016896:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177217.016897:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177217.016898:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177217.016899:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177217.016903:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177217.016904:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177217.016908:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177217.016909:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786177217.016912:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177217.016914:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
00000100:00100000:3.0:1786177217.016918:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:0.0:1786177217.016919:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177217.016929:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1786177217.016930:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177217.016934:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177217.016936:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000100:00000040:3.0:1786177217.016939:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1ef9f45] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1786177217.016943:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000001:3.0:1786177217.016948:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177217.016951:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:0.0:1786177217.016952:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856351232, portal 25
00000100:00080000:3.0:1786177217.016954:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177217.016958:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177217.016958:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177217.016963:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000100:00080000:3.0:1786177217.016963:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177217.016969:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177217.016972:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7440
00000100:00000040:0.0:1786177217.016975:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177217.016977:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177217.016984:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177217.016988:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177217.016994:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177217.016998:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177217.017000:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000040:3.0:1786177217.017001:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:0.0:1786177217.017003:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856351232, offset 0
00000100:00000001:3.0:1786177217.017004:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177217.017009:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177217.017012:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177217.017013:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177217.017017:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177217.017018:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177217.017019:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177217.017021:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:0.0:1786177217.017023:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1786177217.017023:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177217.017025:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177217.017029:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.017032:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786177217.017033:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:3.0:1786177217.017037:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.017039:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786177217.017041:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:3.0:1786177217.017043:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:0.0:1786177217.017048:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:3.0:1786177217.017049:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177217.017054:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177217.017055:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1786177217.017059:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786177217.017060:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:3.0:1786177217.017065:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1786177217.017066:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254915:254915:256:4294967295] 192.168.202.110@tcp LPNI seq info [254904:254904:8:4294967295]
02000000:00000001:3.0:1786177217.017073:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177217.017075:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177217.017078:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177217.017082:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177217.017086:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177217.017087:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177217.017094:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f480.
00000400:00000200:0.0:1786177217.017096:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:3.0:1786177217.017098:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177217.017102:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:0.0:1786177217.017103:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786177217.017104:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177217.017108:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177217.017110:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177217.017112:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1786177217.017113:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
02000000:00000010:3.0:1786177217.017118:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000800:00000200:0.0:1786177217.017122:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177217.017124:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177217.017126:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00100000:3.0:1786177217.017131:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1786177217.017137:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177217.017144:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:3.0:1786177217.017145:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177217.017149:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1786177217.017153:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786177217.017156:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092aa1aa7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:0.0:1786177217.017157:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177217.017165:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1786177217.017166:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177217.017170:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177217.017173:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177217.017176:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:0.0:1786177217.017178:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:3.0:1786177217.017179:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177217.017184:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177217.017188:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7440
00000400:00000200:0.0:1786177217.017189:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000200:3.0:1786177217.017193:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177217.017196:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000400:00000200:3.0:1786177217.017197:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177217.017202:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177217.017204:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177217.017205:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1786177217.017206:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177217.017210:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.017212:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177217.017215:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000200:0.0:1786177217.017216:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177217.017217:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177217.017219:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177217.017221:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177217.017224:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177217.017228:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177217.017235:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.017240:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177217.017243:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177217.017244:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786177217.017245:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177217.017248:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000040:0.0:1786177217.017250:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786177217.017253:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177217.017257:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177217.017261:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177217.017264:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177217.017266:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177217.017267:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.017271:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177217.017272:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177217.017276:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177217.017277:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177217.017279:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.017280:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177217.017282:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f800.
00000020:00000040:3.0:1786177217.017285:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000400:00000200:0.0:1786177217.017289:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786177217.017290:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177217.017292:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.017294:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302116557
00000100:00000001:3.0:1786177217.017295:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177217.017298:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177217.017300:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177217.017302:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
02000000:00000010:3.0:1786177217.017304:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786177217.017308:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.017310:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00100000:3.0:1786177217.017312:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177217.017316:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302116557
00000100:00000001:3.0:1786177217.017319:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000010:0.0:1786177217.017322:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1851349c0.
00000100:00000001:3.0:1786177217.017323:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.017329:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:3.0:1786177217.017330:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4d339d9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.017335:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177217.017341:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177217.017342:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.017345:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00080000:3.0:1786177217.017346:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177217.017349:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177217.017352:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:0.0:1786177217.017353:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177217/real 1786177217]  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786177217.017354:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177217.017360:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177217.017365:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7440
00000400:00000200:3.0:1786177217.017372:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000001:0.0:1786177217.017375:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:3.0:1786177217.017375:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000001:0.0:1786177217.017379:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:3.0:1786177217.017379:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:0.0:1786177217.017383:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000100:00000040:3.0:1786177217.017384:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177217.017388:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000100:00000001:3.0:1786177217.017388:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177217.017391:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:0.0:1786177217.017392:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:3.0:1786177217.017394:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177217.017396:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:3.0:1786177217.017398:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177217.017401:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000200:0.0:1786177217.017402:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177217.017403:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177217.017407:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177217.017411:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177217.017416:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000200:0.0:1786177217.017420:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177217.017421:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.017426:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177217.017430:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177217.017435:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177217.017439:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786177217.017445:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177217.017450:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177217.017454:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177217.017458:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:3.0:1786177217.017458:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177217.017462:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177217.017462:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.017468:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177217.017468:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177217.017470:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786177217.017471:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:3.0:1786177217.017473:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177217.017475:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177217.017477:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1786177217.017480:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ca80.
00000020:00000040:3.0:1786177217.017484:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177217.017488:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177217.017490:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177217.017493:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177217.017495:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177217.017498:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.017499:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:3.0:1786177217.017501:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:3.0:1786177217.017505:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177217.017528:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177217.017533:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177217.017535:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177217.017538:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ffb0865] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177217.017541:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177217.017547:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177217.017553:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7450)
00000100:00000001:0.0:1786177217.017555:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177217.017558:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.017560:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177217.017561:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.017563:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177217.017566:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177217.017568:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177217.017585:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177217.017588:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177217.017593:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177217.017600:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856351232:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177217.017603:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.017605:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177217.017607:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856351232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.017611:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177217.017614:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177217.017617:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177217.017619:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177217.017621:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177217.017622:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177217.017624:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177217.017626:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786177217.017628:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177217.017630:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177217.017631:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:0.0:1786177217.017633:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177217.017634:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.017636:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.017641:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177217.017642:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.017644:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177217.017645:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177217.017646:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177217.017648:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.017653:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177217.017654:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177217.017656:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177217.017657:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177217.017659:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177217.017661:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856351360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177217.017663:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177217.017664:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177217.017665:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177217.017667:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177217.017669:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177217.017670:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177217.017673:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177217.017675:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177217.017678:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177217.017680:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856351360, portal 10
00000100:00000001:0.0:1786177217.017682:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177217.017683:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:0.0:1786177217.017688:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.017692:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177217.017694:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177217.017695:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177217.017697:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856351360, offset 0
00000400:00000200:0.0:1786177217.017699:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177217.017702:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177217.017705:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177217.017707:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177217.017709:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177217.017711:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177217.017713:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177217.017715:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254916:254916:256:4294967295] 192.168.202.110@tcp LPNI seq info [254905:254905:8:4294967295]
00000400:00000200:0.0:1786177217.017718:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177217.017723:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177217.017726:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177217.017729:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177217.017731:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177217.017733:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786177217.017737:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177217.017738:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1786177217.017741:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177217.017742:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177217.017744:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177217.017747:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177217.017749:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177217.017750:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177217.017752:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177217.017754:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.017764:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177217.017766:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.017770:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.017771:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177217.017773:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177217.017776:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177217.017778:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177217.017779:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302112778
00000100:00000001:0.0:1786177217.017781:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302116557
00000100:00000001:0.0:1786177217.017783:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302112778
00000100:00000001:0.0:1786177217.017784:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302116557
00000100:00000001:0.0:1786177217.017786:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177217.017787:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.017789:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177217.017793:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177217/real 1786177217]  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177217.017801:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177217.017803:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177217.017805:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177217.017807:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177217.017808:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177217.017811:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177217.017814:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177217.017824:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.017830:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177217.017832:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.017834:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177217.017836:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177217.017839:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.017846:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177217.017850:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.017857:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177217.017860:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.017862:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177217.017863:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.017865:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177217.017867:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177217.017870:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177217.017873:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177217.017876:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177217.017880:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177217.017886:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856351360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177217.017889:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177217.017891:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.017896:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177217.017897:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177217.017898:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177217.017900:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177217.017901:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177217.017904:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856351488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177217.017906:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177217.017907:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177217.017909:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177217.017910:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177217.017913:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177217.017914:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177217.017916:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177217.017918:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177217.017921:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177217.017923:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856351488, portal 10
00000100:00000001:0.0:1786177217.017924:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177217.017925:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:0.0:1786177217.017929:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.017933:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177217.017935:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177217.017936:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177217.017938:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856351488, offset 0
00000400:00000200:0.0:1786177217.017940:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177217.017943:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177217.017945:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177217.017947:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177217.017949:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177217.017951:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177217.017953:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177217.017955:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254917:254917:256:4294967295] 192.168.202.110@tcp LPNI seq info [254906:254906:8:4294967295]
00000400:00000200:0.0:1786177217.017957:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177217.017961:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177217.017964:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177217.017966:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177217.017969:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177217.017971:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786177217.017974:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177217.017976:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1786177217.017978:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177217.017980:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177217.017982:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177217.017984:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177217.017986:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177217.017988:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177217.017989:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177217.017991:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.017997:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177217.017999:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018003:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.018004:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177217.018006:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177217.018007:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177217.018009:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177217.018011:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302112903
00000100:00000001:0.0:1786177217.018012:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302116557
00000100:00000001:0.0:1786177217.018014:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302112903
00000100:00000001:0.0:1786177217.018015:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302116557
00000100:00000001:0.0:1786177217.018016:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177217.018018:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.018019:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177217.018021:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177217/real 1786177217]  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177217.018026:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177217.018027:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177217.018028:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177217.018030:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177217.018031:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177217.018032:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177217.018034:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177217.018039:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018043:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177217.018043:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018044:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177217.018045:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177217.018047:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018051:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177217.018054:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018058:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177217.018059:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018061:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177217.018062:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018063:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177217.018064:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177217.018065:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177217.018067:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177217.018069:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177217.018071:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177217.018079:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856351488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177217.018082:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177217.018084:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018088:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177217.018089:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177217.018090:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177217.018091:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177217.018092:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177217.018094:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856351616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177217.018096:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177217.018097:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177217.018098:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177217.018099:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177217.018101:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177217.018102:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177217.018104:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177217.018105:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177217.018108:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177217.018110:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856351616, portal 4
00000100:00000001:0.0:1786177217.018111:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177217.018112:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
00000100:00000040:0.0:1786177217.018116:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018120:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177217.018122:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177217.018123:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177217.018124:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856351616, offset 0
00000400:00000200:0.0:1786177217.018127:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177217.018129:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177217.018131:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177217.018134:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177217.018135:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177217.018137:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177217.018139:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177217.018141:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254918:254918:256:4294967295] 192.168.202.110@tcp LPNI seq info [254907:254907:8:4294967295]
00000400:00000200:0.0:1786177217.018144:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177217.018148:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177217.018150:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177217.018152:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177217.018154:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177217.018156:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786177217.018159:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177217.018160:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1786177217.018163:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177217.018164:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177217.018166:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177217.018169:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177217.018172:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177217.018174:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177217.018176:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177217.018180:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018187:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177217.018191:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018198:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.018200:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177217.018203:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177217.018206:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177217.018209:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177217.018212:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302112778
00000100:00000001:0.0:1786177217.018215:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302116557
00000100:00000001:0.0:1786177217.018217:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302112778
00000100:00000001:0.0:1786177217.018219:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302116557
00000100:00000001:0.0:1786177217.018221:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177217.018224:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.018226:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177217.018230:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177217/real 1786177217]  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177217.018235:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177217.018236:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177217.018238:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177217.018239:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177217.018240:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177217.018242:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177217.018244:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177217.018249:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018254:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177217.018254:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018256:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177217.018256:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177217.018259:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018263:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177217.018266:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018270:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177217.018272:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018273:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177217.018274:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018275:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177217.018276:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177217.018278:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177217.018280:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177217.018281:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177217.018283:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177217.018288:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856351616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177217.018290:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177217.018292:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018296:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177217.018297:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177217.018298:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177217.018299:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177217.018301:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177217.018302:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856351744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177217.018304:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177217.018306:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177217.018307:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177217.018308:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177217.018311:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786177217.018312:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177217.018314:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177217.018315:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177217.018317:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177217.018319:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856351744, portal 4
00000100:00000001:0.0:1786177217.018321:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177217.018322:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:0.0:1786177217.018326:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018330:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177217.018332:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177217.018333:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177217.018335:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856351744, offset 0
00000400:00000200:0.0:1786177217.018337:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177217.018339:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177217.018342:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177217.018343:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177217.018345:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177217.018347:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177217.018348:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177217.018350:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254919:254919:256:4294967295] 192.168.202.110@tcp LPNI seq info [254908:254908:8:4294967295]
00000400:00000200:0.0:1786177217.018353:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177217.018358:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177217.018360:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177217.018362:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177217.018365:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177217.018367:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1786177217.018370:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177217.018371:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:0.0:1786177217.018374:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177217.018375:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177217.018377:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177217.018380:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177217.018381:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177217.018383:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177217.018384:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177217.018386:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018396:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177217.018398:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018402:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.018403:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177217.018404:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177217.018407:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177217.018408:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177217.018409:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302112786
00000100:00000001:0.0:1786177217.018411:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302116557
00000100:00000001:0.0:1786177217.018412:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302112786
00000100:00000001:0.0:1786177217.018414:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302116557
00000100:00000001:0.0:1786177217.018415:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177217.018417:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.018418:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177217.018420:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177217/real 1786177217]  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177217.018424:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177217.018425:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177217.018426:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177217.018427:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177217.018428:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177217.018429:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177217.018431:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177217.018436:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018440:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177217.018442:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018443:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177217.018444:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177217.018446:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018450:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177217.018452:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018456:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177217.018458:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018459:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177217.018459:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018460:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177217.018461:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177217.018463:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177217.018464:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177217.018466:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177217.018468:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177217.018472:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856351744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177217.018475:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018476:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177217.018478:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856351360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018482:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177217.018483:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177217.018485:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177217.018488:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177217.018489:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177217.018491:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177217.018493:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177217.018495:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:0.0:1786177217.018498:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177217.018499:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177217.018501:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:0.0:1786177217.018503:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177217.018505:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018506:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177217.018530:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856351488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018538:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177217.018540:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177217.018542:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177217.018543:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177217.018545:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177217.018546:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177217.018547:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177217.018548:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177217.018550:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177217.018550:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177217.018551:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:0.0:1786177217.018553:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177217.018554:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018555:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177217.018558:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856351616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018562:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177217.018563:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177217.018564:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177217.018565:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177217.018566:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177217.018567:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177217.018568:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177217.018569:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786177217.018585:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177217.018586:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177217.018587:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000001:0.0:1786177217.018589:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177217.018590:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018591:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177217.018594:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856351744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177217.018598:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177217.018599:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177217.018600:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786177217.018601:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177217.018602:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177217.018603:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177217.018604:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177217.018605:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786177217.018606:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177217.018607:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177217.018608:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:0.0:1786177217.018609:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177217.018610:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018611:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177217.018614:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177217.018615:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177217.018616:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177217.018617:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177217.592603:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177217.592608:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177217.592612:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177217.592613:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177217.592616:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177217.592617:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177217.592666:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177217.592669:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177217.592672:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177218.040603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177218.040611:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177218.040613:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177218.040619:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177218.040623:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177218.040627:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177218.040630:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177218.040633:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177218.040636:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177218.616664:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177218.616674:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177218.616682:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177218.616683:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177218.616686:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177218.616687:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177218.616747:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177218.616752:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177218.616755:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177219.064622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177219.064629:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177219.064631:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177219.064635:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177219.064638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177219.064642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177219.064644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177219.064646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177219.064648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177219.640602:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177219.640609:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177219.640615:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177219.640617:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177219.640620:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177219.640622:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177219.640688:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177219.640693:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177219.640698:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177220.088662:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177220.088670:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177220.088672:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177220.088677:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177220.088680:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177220.088684:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177220.088687:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177220.088690:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177220.088693:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177220.664632:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177220.664641:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177220.664643:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177220.664647:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177220.664652:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177220.664654:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177220.664656:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177220.664659:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177220.664664:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177220.664666:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177220.664667:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177220.664669:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177220.664670:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177220.664671:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177220.664673:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177220.664676:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177220.664679:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177220.664681:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177220.664729:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177220.664733:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177220.664735:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177220.856615:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302116557, at 4302116608
00000100:00000040:1.0:1786177220.856616:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302116557, at 4302116608
00000100:00000010:1.0:1786177220.856632:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1851349c0.
00000100:00000010:0.0:1786177220.856634:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000800:00000200:3.0:1786177220.984664:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:3.0:1786177220.984674:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:3.0:1786177220.984679:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000001:0.0:1786177220.984853:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000200:1.0:1786177220.984859:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1786177220.984867:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1786177220.984872:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786177220.984875:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177221.112633:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177221.112642:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177221.112644:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177221.112650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177221.112653:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177221.112657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177221.112660:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177221.112663:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177221.112666:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177221.688623:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177221.688628:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177221.688633:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177221.688634:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177221.688636:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177221.688637:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177221.688680:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177221.688683:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177221.688685:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177222.136635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1786177222.136638:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177222.136645:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:3.0:1786177222.136652:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177222.136653:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177222.136659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177222.136659:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177222.136664:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177222.136665:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177222.136675:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177222.136675:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786177222.136686:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1786177222.136691:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7445
00000100:00000001:0.0:1786177222.136692:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.136700:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177222.136706:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177222.136713:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177222.136718:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177222.136721:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.136725:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177222.136727:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.136729:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177222.136734:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177222.136736:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177222.136739:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177222.136743:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177222.136745:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177222.136746:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177222.136749:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.136750:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177222.136752:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.136753:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177222.136756:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177222.136758:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177222.136760:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177222.136762:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177222.136766:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177222.136768:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177222.136769:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177222.136771:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177222.136781:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69fb80.
00000020:00000040:3.0:1786177222.136784:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177222.136789:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177222.136790:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177222.136793:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177222.136797:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177222.136801:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177222.136808:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000001:3.0:1786177222.136816:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177222.136825:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177222.136834:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177222.136839:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177222.136843:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014f55bbb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177222.136863:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177222.136866:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177222.136868:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177222.136869:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177222.136870:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177222.136873:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177222.136874:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7445
00000400:00000200:3.0:1786177222.136877:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177222.136879:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177222.136881:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177222.136883:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.136886:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177222.136889:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.136891:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177222.136892:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177222.136893:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177222.136894:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177222.136896:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:0.0:1786177222.136897:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1786177222.136899:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786177222.136902:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1786177222.136903:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786177222.136906:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177222.136907:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.136915:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177222.136917:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.136918:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177222.136921:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786177222.136926:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:0.0:1786177222.136928:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177222.136934:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177222.136942:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177222.136952:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1786177222.136952:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177222.136956:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177222.136959:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177222.136963:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1786177222.136964:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:0.0:1786177222.136971:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:3.0:1786177222.136973:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1786177222.136977:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1786177222.136980:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177222.136988:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177222.136989:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856351872:192.168.202.110@tcp:250:kworker.0
00000100:00000010:3.0:1786177222.136995:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d880.
00000100:00000001:0.0:1786177222.136999:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:3.0:1786177222.136999:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177222.137006:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177222.137008:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177222.137010:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1786177222.137011:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177222.137016:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177222.137018:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177222.137019:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177222.137028:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177222.137029:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000010:0.0:1786177222.137038:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786177222.137042:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177222.137042:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177222.137051:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:3.0:1786177222.137055:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:0.0:1786177222.137058:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000100:00000001:3.0:1786177222.137069:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177222.137071:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177222.137076:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097f9b8db] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1786177222.137083:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000001:3.0:1786177222.137085:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177222.137088:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:0.0:1786177222.137091:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856351872, portal 25
00000100:00080000:3.0:1786177222.137094:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177222.137096:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177222.137097:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177222.137099:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00080000:3.0:1786177222.137101:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177222.137110:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177222.137120:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7445
00000100:00000040:0.0:1786177222.137121:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177222.137139:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177222.137151:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1786177222.137152:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1786177222.137156:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177222.137162:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786177222.137166:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:0.0:1786177222.137169:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000001:3.0:1786177222.137174:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:0.0:1786177222.137175:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856351872, offset 0
00000100:00000040:3.0:1786177222.137180:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.137184:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177222.137186:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:0.0:1786177222.137192:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:3.0:1786177222.137192:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177222.137201:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177222.137205:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:0.0:1786177222.137214:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1786177222.137215:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177222.137222:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177222.137227:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177222.137230:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:3.0:1786177222.137233:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177222.137240:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177222.137241:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1786177222.137246:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786177222.137252:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000040:3.0:1786177222.137253:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:0.0:1786177222.137261:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00080000:3.0:1786177222.137262:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1786177222.137266:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.137277:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177222.137280:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254920:254920:256:4294967295] 192.168.202.110@tcp LPNI seq info [254909:254909:8:4294967295]
02000000:00000001:3.0:1786177222.137287:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786177222.137291:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786177222.137294:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177222.137304:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177222.137309:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786177222.137311:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177222.137320:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000800:00000200:0.0:1786177222.137330:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000008:00000001:3.0:1786177222.137331:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177222.137339:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1786177222.137342:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000100:00000010:3.0:1786177222.137343:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c380.
00000020:00000040:3.0:1786177222.137356:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177222.137360:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177222.137362:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177222.137363:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177222.137367:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:0.0:1786177222.137370:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
02000000:00000001:3.0:1786177222.137370:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177222.137379:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786177222.137388:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:3.0:1786177222.137392:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000800:00000200:0.1:1786177222.137392:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:3.0:1786177222.137401:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1786177222.137406:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177222.137410:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000100:00100000:3.0:1786177222.137413:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1786177222.137417:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1786177222.137431:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1786177222.137432:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1786177222.137438:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177222.137440:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177222.137443:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000040:3.0:1786177222.137448:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000833bf1c9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.137449:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177222.137455:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177222.137550:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177222.137552:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177222.137555:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177222.137556:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177222.137560:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177222.137565:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786177222.137567:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1786177222.137571:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7445
00000100:00000040:0.0:1786177222.137579:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177222.137589:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177222.137605:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1786177222.137606:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.137617:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:3.0:1786177222.137617:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177222.137631:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177222.137634:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177222.137634:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177222.137646:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177222.137649:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177222.137652:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177222.137654:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302121678
00000020:00000001:3.0:1786177222.137659:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177222.137662:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177222.137664:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000010:0.0:1786177222.137667:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000020:00000040:3.0:1786177222.137668:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177222.137673:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177222.137676:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177222.137681:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177222.137682:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177222.137689:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177222.137693:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177222.137696:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302121678
00000100:00000001:3.0:1786177222.137697:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177222.137703:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177222.137713:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000010:0.0:1786177222.137714:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134f00.
00000100:00000001:3.0:1786177222.137720:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177222.137724:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177222.137725:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
02000000:00000001:3.0:1786177222.137728:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177222.137730:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177222.137732:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.137736:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177222.137737:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177222.137745:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786177222.137746:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000008:00000001:3.0:1786177222.137750:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177222.137757:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177222/real 1786177222]  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177222.137757:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177222.137767:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ed80.
00000020:00000040:3.0:1786177222.137777:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000400:00000001:0.0:1786177222.137779:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:3.0:1786177222.137783:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000001:0.0:1786177222.137784:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:3.0:1786177222.137789:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177222.137791:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000100:00000001:3.0:1786177222.137803:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1786177222.137805:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
02000000:00000001:3.0:1786177222.137806:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:0.0:1786177222.137811:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
02000000:00000001:3.0:1786177222.137812:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.137818:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000010:3.0:1786177222.137820:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000200:0.0:1786177222.137826:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177222.137828:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177222.137843:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1786177222.137856:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177222.137860:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177222.137864:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177222.137873:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ccd6c6d2] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.137891:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177222.137893:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177222.137899:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177222.137900:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786177222.137911:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177222.137913:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177222.137914:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177222.137921:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786177222.137921:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177222.137935:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177222.137940:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7445
00000100:00100000:0.0:1786177222.137944:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177222.137950:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177222.137963:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177222.137972:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177222.137977:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177222.137982:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.137991:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177222.137992:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177222.137997:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.138003:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177222.138008:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177222.138010:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177222.138015:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177222.138022:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177222.138029:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:0.0:1786177222.138032:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:3.0:1786177222.138033:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177222.138038:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.138044:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177222.138045:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177222.138046:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177222.138051:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177222.138053:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:3.0:1786177222.138055:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:0.0:1786177222.138059:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786177222.138060:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177222.138069:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.138070:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000001:3.0:1786177222.138071:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177222.138073:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177222.138075:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:3.0:1786177222.138077:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177222.138082:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00080000:0.0:1786177222.138083:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000008:00100000:3.0:1786177222.138091:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00080000:0.0:1786177222.138096:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000008:00000001:3.0:1786177222.138102:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.138106:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1786177222.138107:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177222.138115:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e680.
00000100:00000040:0.0:1786177222.138116:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177222.138123:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177222.138132:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177222.138137:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177222.138142:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:0.0:1786177222.138143:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856351872:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1786177222.138147:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177222.138152:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.138155:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.138159:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000010:3.0:1786177222.138159:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000040:0.0:1786177222.138165:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856351872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177222.138166:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177222.138182:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177222.138183:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177222.138188:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177222.138193:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
00000100:00000001:3.0:1786177222.138196:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:0.0:1786177222.138199:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:3.0:1786177222.138200:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177222.138204:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177222.138207:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177222.138211:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00000040:3.0:1786177222.138212:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1839131] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786177222.138214:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786177222.138221:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177222.138223:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177222.138226:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:0.0:1786177222.138233:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177222.138237:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.138245:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177222.138247:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.138252:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177222.138254:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.138257:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1786177222.138257:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7455)
00000100:00000001:0.0:1786177222.138263:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177222.138266:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177222.138271:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138278:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177222.138280:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177222.138283:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177222.138285:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177222.138287:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177222.138290:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177222.138293:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177222.138295:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177222.138296:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177222.138297:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177222.138300:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786177222.138302:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177222.138305:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177222.138307:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177222.138310:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177222.138313:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856352000, portal 10
00000100:00000001:0.0:1786177222.138314:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177222.138315:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
00000100:00000040:0.0:1786177222.138320:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138325:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177222.138326:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177222.138328:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177222.138329:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856352000, offset 0
00000400:00000200:0.0:1786177222.138332:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177222.138335:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177222.138338:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177222.138340:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177222.138342:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177222.138344:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177222.138346:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177222.138348:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254921:254921:256:4294967295] 192.168.202.110@tcp LPNI seq info [254910:254910:8:4294967295]
00000400:00000200:0.0:1786177222.138351:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177222.138356:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177222.138359:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177222.138361:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177222.138365:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177222.138367:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786177222.138370:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177222.138372:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1786177222.138375:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177222.138376:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177222.138378:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177222.138381:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177222.138383:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177222.138384:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177222.138386:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177222.138388:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138394:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177222.138395:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138400:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.138401:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177222.138403:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177222.138407:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177222.138409:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177222.138411:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302117898
00000100:00000001:0.0:1786177222.138412:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302121678
00000100:00000001:0.0:1786177222.138414:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302117898
00000100:00000001:0.0:1786177222.138416:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302121678
00000100:00000001:0.0:1786177222.138417:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177222.138419:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.138420:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177222.138422:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177222/real 1786177222]  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177222.138428:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177222.138429:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177222.138430:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177222.138431:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177222.138432:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177222.138434:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177222.138435:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177222.138440:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138444:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177222.138445:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.138447:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177222.138448:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177222.138450:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138454:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177222.138456:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138460:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177222.138462:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.138463:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177222.138463:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.138465:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177222.138466:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177222.138468:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177222.138471:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177222.138472:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177222.138475:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177222.138480:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177222.138482:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177222.138484:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138488:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177222.138489:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177222.138491:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177222.138492:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177222.138493:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177222.138495:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177222.138496:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177222.138498:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177222.138498:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177222.138500:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177222.138501:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177222.138503:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177222.138505:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177222.138506:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177222.138508:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177222.138542:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856352128, portal 10
00000100:00000001:0.0:1786177222.138545:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177222.138547:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764293504 : -114547945258112 : ffff97d1ba69c380)
00000100:00000040:0.0:1786177222.138553:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138562:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177222.138564:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177222.138566:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177222.138568:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856352128, offset 0
00000400:00000200:0.0:1786177222.138570:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177222.138573:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177222.138576:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177222.138578:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177222.138580:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177222.138582:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177222.138583:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177222.138585:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254922:254922:256:4294967295] 192.168.202.110@tcp LPNI seq info [254911:254911:8:4294967295]
00000400:00000200:0.0:1786177222.138588:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177222.138592:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177222.138596:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177222.138598:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177222.138600:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177222.138602:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786177222.138605:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177222.138607:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1786177222.138609:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177222.138611:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177222.138613:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177222.138615:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177222.138617:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177222.138618:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177222.138620:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177222.138622:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138628:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177222.138630:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138638:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.138639:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177222.138641:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177222.138643:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177222.138646:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177222.138647:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302118024
00000100:00000001:0.0:1786177222.138649:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302121678
00000100:00000001:0.0:1786177222.138650:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302118024
00000100:00000001:0.0:1786177222.138652:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302121678
00000100:00000001:0.0:1786177222.138653:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177222.138654:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.138655:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177222.138658:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177222/real 1786177222]  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177222.138663:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177222.138664:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177222.138666:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177222.138668:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177222.138670:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177222.138672:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177222.138675:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177222.138684:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138690:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177222.138692:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.138694:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177222.138695:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177222.138700:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138708:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177222.138712:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138720:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177222.138722:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.138724:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177222.138725:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.138726:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177222.138727:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177222.138729:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177222.138732:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177222.138733:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177222.138736:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177222.138742:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177222.138744:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177222.138747:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138751:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177222.138752:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177222.138754:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177222.138755:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177222.138756:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177222.138759:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177222.138761:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177222.138762:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177222.138763:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177222.138765:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177222.138767:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177222.138768:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177222.138771:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177222.138773:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177222.138776:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177222.138778:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856352256, portal 4
00000100:00000001:0.0:1786177222.138779:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177222.138780:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:0.0:1786177222.138785:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138789:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177222.138790:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177222.138792:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177222.138793:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856352256, offset 0
00000400:00000200:0.0:1786177222.138796:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177222.138798:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177222.138801:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177222.138803:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177222.138804:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177222.138806:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177222.138808:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177222.138810:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254923:254923:256:4294967295] 192.168.202.110@tcp LPNI seq info [254912:254912:8:4294967295]
00000400:00000200:0.0:1786177222.138813:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177222.138816:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177222.138819:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177222.138821:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177222.138823:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177222.138826:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786177222.138828:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177222.138830:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1786177222.138833:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177222.138834:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177222.138836:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177222.138838:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177222.138840:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177222.138842:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177222.138843:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177222.138846:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138851:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177222.138853:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138857:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.138858:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177222.138860:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177222.138863:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177222.138865:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177222.138866:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302117899
00000100:00000001:0.0:1786177222.138868:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302121678
00000100:00000001:0.0:1786177222.138869:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302117899
00000100:00000001:0.0:1786177222.138871:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302121678
00000100:00000001:0.0:1786177222.138872:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177222.138873:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.138874:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177222.138876:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177222/real 1786177222]  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177222.138881:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177222.138882:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177222.138883:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177222.138884:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177222.138885:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177222.138887:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177222.138889:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177222.138894:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138900:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177222.138901:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.138903:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177222.138904:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177222.138908:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138915:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177222.138920:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138928:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177222.138930:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.138932:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177222.138933:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.138935:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177222.138937:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177222.138939:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177222.138943:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177222.138945:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177222.138949:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177222.138957:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177222.138961:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177222.138964:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.138972:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177222.138973:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177222.138976:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177222.138977:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177222.138979:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177222.138982:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177222.138985:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177222.138987:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177222.138988:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177222.138991:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177222.138994:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177222.138996:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177222.138999:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177222.139001:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177222.139004:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177222.139007:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856352384, portal 4
00000100:00000001:0.0:1786177222.139009:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177222.139011:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00000040:0.0:1786177222.139016:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.139023:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177222.139025:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177222.139027:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177222.139029:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856352384, offset 0
00000400:00000200:0.0:1786177222.139033:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177222.139036:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177222.139040:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177222.139042:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177222.139045:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177222.139047:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177222.139050:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177222.139054:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254924:254924:256:4294967295] 192.168.202.110@tcp LPNI seq info [254913:254913:8:4294967295]
00000400:00000200:0.0:1786177222.139058:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177222.139065:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177222.139070:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177222.139073:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177222.139077:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177222.139080:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1786177222.139085:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177222.139088:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1786177222.139091:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177222.139094:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177222.139097:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177222.139100:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177222.139102:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177222.139103:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177222.139105:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177222.139107:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.139113:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177222.139115:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.139119:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.139120:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177222.139123:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177222.139125:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177222.139127:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177222.139129:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302117907
00000100:00000001:0.0:1786177222.139131:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302121678
00000100:00000001:0.0:1786177222.139132:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302117907
00000100:00000001:0.0:1786177222.139134:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302121678
00000100:00000001:0.0:1786177222.139135:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177222.139137:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.139138:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177222.139141:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177222/real 1786177222]  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177222.139146:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177222.139147:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177222.139148:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177222.139150:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177222.139151:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177222.139152:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177222.139154:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177222.139159:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.139163:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177222.139164:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.139165:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177222.139166:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177222.139168:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.139172:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177222.139174:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.139179:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177222.139180:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.139181:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177222.139182:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.139183:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177222.139185:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177222.139186:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177222.139188:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177222.139190:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177222.139192:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177222.139197:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177222.139200:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.139201:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177222.139203:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856352000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.139207:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177222.139208:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177222.139209:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786177222.139211:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177222.139212:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177222.139213:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177222.139214:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177222.139215:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786177222.139217:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177222.139218:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177222.139219:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:0.0:1786177222.139221:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177222.139222:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.139223:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177222.139225:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c380 x1872934856352128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.139229:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177222.139229:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177222.139230:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177222.139232:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177222.139233:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177222.139234:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177222.139235:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177222.139236:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786177222.139238:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177222.139239:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177222.139240:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c380.
00000100:00000001:0.0:1786177222.139241:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177222.139242:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.139243:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177222.139245:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856352256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.139249:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177222.139249:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177222.139250:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177222.139251:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177222.139252:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177222.139253:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177222.139254:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177222.139255:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786177222.139256:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177222.139256:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177222.139257:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:0.0:1786177222.139259:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177222.139259:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.139260:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177222.139262:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856352384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177222.139266:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177222.139266:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177222.139267:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177222.139268:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177222.139269:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177222.139269:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177222.139270:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177222.139271:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177222.139273:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177222.139273:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177222.139274:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:0.0:1786177222.139276:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177222.139277:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.139278:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177222.139281:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177222.139282:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177222.139283:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177222.139284:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177222.712653:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177222.712660:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177222.712666:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177222.712667:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177222.712669:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177222.712671:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177222.712742:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177222.712746:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177222.712749:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177223.160654:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177223.160663:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177223.160665:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177223.160671:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177223.160675:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177223.160679:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177223.160682:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177223.160685:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177223.160688:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177223.736619:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177223.736624:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177223.736628:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177223.736629:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177223.736631:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177223.736633:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177223.736635:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177223.736637:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177223.736639:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177223.736641:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177223.736643:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177223.736644:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177223.736693:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177223.736696:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177223.736699:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177224.184657:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177224.184665:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177224.184667:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177224.184673:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177224.184676:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177224.184679:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177224.184682:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177224.184684:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177224.184687:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177224.760592:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177224.760597:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177224.760601:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177224.760602:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177224.760604:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177224.760605:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177224.760655:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177224.760660:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177224.760663:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177225.208582:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177225.208587:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177225.208589:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177225.208593:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177225.208596:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177225.208599:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177225.208601:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177225.208602:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177225.208604:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177225.784605:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177225.784609:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177225.784613:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177225.784614:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177225.784616:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177225.784616:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177225.784667:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177225.784670:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177225.784672:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177225.976611:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302121678, at 4302121728
00000100:00000040:1.0:1786177225.976611:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302121678, at 4302121728
00000100:00000010:0.0:1786177225.976625:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000010:1.0:1786177225.976627:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134f00.
00000100:00000001:0.0:1786177226.232618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177226.232625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177226.232627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177226.232632:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177226.232636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177226.232640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177226.232643:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177226.232646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177226.232649:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177226.808565:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177226.808570:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177226.808573:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177226.808574:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177226.808577:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177226.808578:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177226.808629:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177226.808632:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177226.808634:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786177227.256572:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177227.256582:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177227.256584:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177227.256588:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177227.256591:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786177227.256592:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177227.256596:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177227.256598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177227.256599:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7450
00000100:00100000:0.0:1786177227.256601:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177227.256603:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177227.256607:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1786177227.256607:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177227.256609:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177227.256612:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1786177227.256613:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1786177227.256615:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177227.256615:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177227.256618:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177227.256621:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177227.256624:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177227.256626:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177227.256630:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177227.256632:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177227.256635:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177227.256637:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177227.256640:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177227.256642:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177227.256645:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177227.256647:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177227.256649:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177227.256650:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177227.256653:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177227.256655:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177227.256656:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177227.256659:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177227.256661:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177227.256663:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177227.256664:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177227.256666:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177227.256673:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69dc00.
00000020:00000040:3.0:1786177227.256675:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177227.256678:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177227.256679:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177227.256681:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177227.256682:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177227.256686:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177227.256691:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786177227.256696:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177227.256702:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177227.256709:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177227.256712:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177227.256715:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097fe80e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177227.256730:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177227.256732:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177227.256736:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177227.256737:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177227.256738:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177227.256741:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177227.256743:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7450
00000400:00000200:3.0:1786177227.256747:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177227.256750:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177227.256752:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177227.256753:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1786177227.256754:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177227.256756:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177227.256756:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177227.256758:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177227.256758:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177227.256760:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177227.256762:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000040:0.0:1786177227.256763:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177227.256763:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177227.256765:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177227.256767:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786177227.256769:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786177227.256771:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000001:3.0:1786177227.256771:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:0.0:1786177227.256772:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177227.256774:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:0.0:1786177227.256776:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1786177227.256776:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177227.256777:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:0.0:1786177227.256778:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177227.256779:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:0.0:1786177227.256781:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177227.256781:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00100000:0.0:1786177227.256784:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856352512:192.168.202.110@tcp:250:kworker.0
00000100:00080000:3.0:1786177227.256784:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177227.256786:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177227.256788:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177227.256793:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177227.256794:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177227.256795:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177227.256796:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:0.0:1786177227.256800:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:3.0:1786177227.256800:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1786177227.256802:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1786177227.256802:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177227.256804:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177227.256805:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:3.0:1786177227.256809:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e680.
02000000:00000010:0.0:1786177227.256810:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177227.256812:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177227.256812:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177227.256814:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177227.256815:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177227.256816:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:0.0:1786177227.256817:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786177227.256818:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177227.256819:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177227.256821:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
02000000:00000010:3.0:1786177227.256823:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:3.0:1786177227.256826:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177227.256827:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00100000:3.0:1786177227.256830:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1786177227.256833:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856352512, portal 25
00000100:00000001:0.0:1786177227.256835:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177227.256836:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177227.256837:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764299776 : -114547945251840 : ffff97d1ba69dc00)
00000100:00000001:3.0:1786177227.256838:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177227.256841:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1839131] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177227.256842:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.256850:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786177227.256850:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177227.256852:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1786177227.256853:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177227.256855:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00080000:3.0:1786177227.256855:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177227.256857:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:0.0:1786177227.256858:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856352512, offset 0
00000100:00080000:3.0:1786177227.256859:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177227.256861:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177227.256863:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7450
00000400:00000200:0.0:1786177227.256864:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177227.256866:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177227.256868:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:0.0:1786177227.256870:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:3.0:1786177227.256870:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177227.256873:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177227.256875:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177227.256880:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1786177227.256882:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:0.0:1786177227.256884:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:3.0:1786177227.256884:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177227.256886:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786177227.256887:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:3.0:1786177227.256888:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177227.256889:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000200:0.0:1786177227.256890:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000001:3.0:1786177227.256891:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:0.0:1786177227.256893:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254925:254925:256:4294967295] 192.168.202.110@tcp LPNI seq info [254914:254914:8:4294967295]
00000020:00000040:3.0:1786177227.256893:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786177227.256894:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177227.256896:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:0.0:1786177227.256898:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:3.0:1786177227.256899:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177227.256901:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177227.256902:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177227.256904:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786177227.256906:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000040:3.0:1786177227.256906:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786177227.256909:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177227.256911:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177227.256912:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786177227.256912:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177227.256914:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000010:0.0:1786177227.256916:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
02000000:00000001:3.0:1786177227.256916:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177227.256917:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177227.256919:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177227.256921:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177227.256922:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177227.256923:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177227.256925:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000100:00000010:3.0:1786177227.256926:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ed80.
00000020:00000040:3.0:1786177227.256928:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177227.256930:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177227.256931:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786177227.256931:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177227.256933:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177227.256934:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:0.1:1786177227.256934:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
02000000:00000001:3.0:1786177227.256935:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1786177227.256938:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000010:3.0:1786177227.256939:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000800:00000010:0.0:1786177227.256941:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000100:00000001:3.0:1786177227.256942:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177227.256946:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:3.0:1786177227.256946:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1786177227.256951:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1786177227.256952:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177227.256954:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177227.256955:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177227.256957:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000040:3.0:1786177227.256958:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ccd6c6d2] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.256961:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177227.256965:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177227.256965:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177227.256967:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177227.256971:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177227.256972:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177227.256973:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1786177227.256974:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:0.0:1786177227.256976:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177227.256977:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177227.256979:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7450
00000400:00000200:3.0:1786177227.256982:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177227.256983:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.256985:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000200:3.0:1786177227.256985:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177227.256987:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177227.256989:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177227.256990:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177227.256990:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177227.256992:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177227.256994:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177227.256994:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177227.256996:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302126797
00000020:00000001:3.0:1786177227.256996:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177227.256997:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177227.256998:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000010:0.0:1786177227.257000:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000020:00000040:3.0:1786177227.257000:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177227.257002:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177227.257003:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:0.0:1786177227.257006:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000040:3.0:1786177227.257006:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177227.257007:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177227.257009:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177227.257010:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302126797
00000100:00000001:3.0:1786177227.257011:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000010:0.0:1786177227.257013:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134420.
00000020:00000040:3.0:1786177227.257013:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:0.0:1786177227.257016:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00080000:3.0:1786177227.257016:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177227.257018:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.257019:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177227.257019:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177227.257021:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177227.257023:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177227.257025:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177227.257027:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786177227.257029:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1786177227.257029:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177227.257030:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177227.257033:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c380.
00000100:00000001:0.0:1786177227.257034:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177227.257038:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177227/real 1786177227]  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177227.257040:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177227.257043:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177227.257046:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1786177227.257047:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177227.257048:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1786177227.257048:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177227.257049:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:0.0:1786177227.257050:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
02000000:00000001:3.0:1786177227.257051:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177227.257052:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177227.257054:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
02000000:00000010:3.0:1786177227.257054:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:0.0:1786177227.257057:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:3.0:1786177227.257057:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786177227.257060:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177227.257061:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177227.257068:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000200:0.0:1786177227.257069:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177227.257070:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177227.257073:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000833bf1c9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257076:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177227.257078:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257080:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177227.257081:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177227.257081:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177227.257083:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:0.0:1786177227.257085:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786177227.257086:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177227.257088:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177227.257089:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177227.257092:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177227.257092:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177227.257094:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7450
00000100:00000040:0.0:1786177227.257096:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177227.257097:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177227.257100:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177227.257102:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177227.257103:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:3.0:1786177227.257104:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177227.257106:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177227.257106:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177227.257108:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:3.0:1786177227.257108:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177227.257109:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177227.257110:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177227.257111:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000001:3.0:1786177227.257112:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:0.0:1786177227.257113:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:3.0:1786177227.257113:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177227.257115:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00080000:0.0:1786177227.257116:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000020:00000040:3.0:1786177227.257116:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177227.257118:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00080000:0.0:1786177227.257119:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000020:00000001:3.0:1786177227.257120:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:0.0:1786177227.257121:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177227.257123:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:0.0:1786177227.257125:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177227.257125:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177227.257126:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177227.257128:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177227.257130:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00100000:0.0:1786177227.257133:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856352512:192.168.202.110@tcp:250:kworker.0
00000100:00080000:3.0:1786177227.257133:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177227.257135:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177227.257136:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177227.257137:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177227.257138:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177227.257139:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1786177227.257140:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177227.257141:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:0.0:1786177227.257142:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69dc00 x1872934856352512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1786177227.257143:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177227.257145:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177227.257147:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177227.257151:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d880.
00000020:00000040:3.0:1786177227.257153:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:0.0:1786177227.257154:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177227.257155:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177227.257156:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000001:3.0:1786177227.257156:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177227.257157:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177227.257158:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:0.0:1786177227.257159:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:3.0:1786177227.257160:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177227.257161:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177227.257163:0:2391:0:(genops.c:1238:class_import_put()) Process entered
02000000:00000010:3.0:1786177227.257163:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000020:00000040:0.0:1786177227.257164:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:3.0:1786177227.257165:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177227.257166:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00100000:3.0:1786177227.257168:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:0.0:1786177227.257169:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:0.0:1786177227.257172:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177227.257173:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000001:3.0:1786177227.257174:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000010:0.0:1786177227.257175:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69dc00.
00000100:00000001:3.0:1786177227.257176:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.257178:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177227.257179:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177227.257179:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097f9b8db] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257181:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177227.257185:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.257187:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177227.257188:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177227.257189:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7460)
00000100:00000001:0.0:1786177227.257190:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177227.257191:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177227.257192:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177227.257195:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257201:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177227.257203:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177227.257205:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177227.257207:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177227.257208:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177227.257212:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177227.257214:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177227.257216:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177227.257218:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177227.257219:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177227.257222:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177227.257224:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177227.257227:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177227.257230:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177227.257233:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177227.257235:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856352640, portal 10
00000100:00000001:0.0:1786177227.257236:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177227.257237:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00000040:0.0:1786177227.257242:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257246:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177227.257247:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177227.257248:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177227.257250:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856352640, offset 0
00000400:00000200:0.0:1786177227.257252:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177227.257254:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177227.257257:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177227.257258:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177227.257260:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177227.257261:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177227.257263:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177227.257265:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254926:254926:256:4294967295] 192.168.202.110@tcp LPNI seq info [254915:254915:8:4294967295]
00000400:00000200:0.0:1786177227.257267:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177227.257271:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177227.257274:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177227.257276:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177227.257278:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177227.257280:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786177227.257282:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177227.257284:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1786177227.257286:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177227.257287:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177227.257289:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177227.257291:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177227.257293:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177227.257294:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177227.257296:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177227.257298:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257303:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177227.257304:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257308:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.257309:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177227.257311:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177227.257313:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177227.257315:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177227.257316:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302123017
00000100:00000001:0.0:1786177227.257318:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302126797
00000100:00000001:0.0:1786177227.257320:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302123017
00000100:00000001:0.0:1786177227.257321:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302126797
00000100:00000001:0.0:1786177227.257323:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177227.257325:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.257327:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177227.257331:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177227/real 1786177227]  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177227.257337:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177227.257339:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177227.257340:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177227.257342:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177227.257344:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177227.257346:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177227.257349:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177227.257357:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257367:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177227.257368:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257370:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177227.257371:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177227.257375:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257381:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177227.257384:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257390:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177227.257393:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257394:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177227.257395:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257397:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177227.257398:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177227.257399:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177227.257401:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177227.257403:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177227.257405:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177227.257410:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177227.257412:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177227.257414:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257418:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177227.257418:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177227.257420:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177227.257421:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177227.257423:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177227.257424:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177227.257426:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177227.257427:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177227.257428:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177227.257429:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177227.257431:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177227.257432:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177227.257435:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177227.257436:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177227.257438:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177227.257440:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856352768, portal 10
00000100:00000001:0.0:1786177227.257441:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177227.257442:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:0.0:1786177227.257446:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257449:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177227.257451:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177227.257452:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177227.257453:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856352768, offset 0
00000400:00000200:0.0:1786177227.257455:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177227.257457:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177227.257459:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177227.257461:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177227.257463:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177227.257464:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177227.257465:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177227.257467:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254927:254927:256:4294967295] 192.168.202.110@tcp LPNI seq info [254916:254916:8:4294967295]
00000400:00000200:0.0:1786177227.257469:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177227.257496:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177227.257499:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177227.257502:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177227.257504:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177227.257506:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786177227.257508:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177227.257524:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1786177227.257527:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177227.257528:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177227.257530:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177227.257533:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177227.257534:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177227.257535:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177227.257537:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177227.257539:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257544:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177227.257546:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257549:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.257550:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177227.257552:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177227.257554:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177227.257556:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177227.257557:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302123143
00000100:00000001:0.0:1786177227.257559:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302126797
00000100:00000001:0.0:1786177227.257560:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302123143
00000100:00000001:0.0:1786177227.257561:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302126797
00000100:00000001:0.0:1786177227.257563:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177227.257564:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.257565:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177227.257567:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177227/real 1786177227]  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177227.257571:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177227.257572:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177227.257573:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177227.257575:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177227.257576:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177227.257577:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177227.257578:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177227.257583:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257586:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177227.257587:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257588:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177227.257589:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177227.257590:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257594:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177227.257596:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257600:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177227.257601:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257602:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177227.257603:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257604:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177227.257605:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177227.257606:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177227.257608:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177227.257609:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177227.257612:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177227.257616:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177227.257618:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177227.257620:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257623:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177227.257624:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177227.257625:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177227.257626:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177227.257627:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177227.257629:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177227.257630:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177227.257631:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177227.257632:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177227.257633:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177227.257635:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177227.257636:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177227.257637:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177227.257639:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177227.257641:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177227.257643:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856352896, portal 4
00000100:00000001:0.0:1786177227.257644:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177227.257644:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764293504 : -114547945258112 : ffff97d1ba69c380)
00000100:00000040:0.0:1786177227.257648:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257652:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177227.257653:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177227.257654:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177227.257656:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856352896, offset 0
00000400:00000200:0.0:1786177227.257658:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177227.257660:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177227.257662:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177227.257664:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177227.257666:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177227.257667:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177227.257668:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177227.257670:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254928:254928:256:4294967295] 192.168.202.110@tcp LPNI seq info [254917:254917:8:4294967295]
00000400:00000200:0.0:1786177227.257673:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177227.257676:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177227.257678:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177227.257680:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177227.257682:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177227.257683:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786177227.257686:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177227.257687:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1786177227.257689:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177227.257691:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177227.257692:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177227.257694:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177227.257696:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177227.257696:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177227.257698:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177227.257700:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257704:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177227.257705:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257709:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.257710:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177227.257711:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177227.257713:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177227.257714:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177227.257716:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302123018
00000100:00000001:0.0:1786177227.257717:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302126797
00000100:00000001:0.0:1786177227.257718:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302123018
00000100:00000001:0.0:1786177227.257719:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302126797
00000100:00000001:0.0:1786177227.257720:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177227.257721:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.257722:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177227.257724:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177227/real 1786177227]  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177227.257728:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177227.257728:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177227.257729:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177227.257730:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177227.257731:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177227.257732:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177227.257734:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177227.257738:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257742:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177227.257742:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257743:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177227.257744:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177227.257746:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257749:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177227.257751:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257754:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177227.257756:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257757:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177227.257758:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257759:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177227.257760:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177227.257761:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177227.257763:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177227.257764:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177227.257766:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177227.257770:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856352896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177227.257771:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177227.257773:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257776:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177227.257777:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177227.257778:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177227.257779:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177227.257780:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177227.257781:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856353024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177227.257783:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177227.257784:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177227.257784:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177227.257785:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177227.257787:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786177227.257788:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177227.257789:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177227.257791:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177227.257792:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177227.257794:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856353024, portal 4
00000100:00000001:0.0:1786177227.257795:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177227.257796:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
00000100:00000040:0.0:1786177227.257799:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257803:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177227.257804:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177227.257805:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177227.257806:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856353024, offset 0
00000400:00000200:0.0:1786177227.257808:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177227.257810:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177227.257812:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177227.257814:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177227.257816:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177227.257818:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177227.257820:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177227.257823:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254929:254929:256:4294967295] 192.168.202.110@tcp LPNI seq info [254918:254918:8:4294967295]
00000400:00000200:0.0:1786177227.257826:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177227.257832:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177227.257836:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177227.257839:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177227.257842:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177227.257845:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1786177227.257848:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177227.257851:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:0.0:1786177227.257854:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177227.257856:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177227.257858:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177227.257861:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177227.257864:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177227.257865:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177227.257868:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177227.257871:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257883:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177227.257886:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177252 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257892:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.257893:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177227.257896:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177227.257899:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177227.257902:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177227.257904:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302123026
00000100:00000001:0.0:1786177227.257906:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302126797
00000100:00000001:0.0:1786177227.257908:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302123026
00000100:00000001:0.0:1786177227.257910:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302126797
00000100:00000001:0.0:1786177227.257912:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177227.257914:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.257916:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177227.257919:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177227/real 1786177227]  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177227.257926:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177227.257927:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177227.257929:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177227.257931:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177227.257933:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177227.257935:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177227.257938:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177227.257946:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257952:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177227.257953:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257955:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177227.257956:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177227.257960:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257966:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177227.257970:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.257977:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177227.257979:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257980:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177227.257981:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.257983:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177227.257984:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177227.257986:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177227.257989:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177227.257991:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177227.257995:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177227.258003:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856353024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177227.258006:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.258007:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177227.258011:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856352640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.258017:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177227.258018:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177227.258020:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177227.258022:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177227.258024:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177227.258025:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177227.258027:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177227.258029:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786177227.258031:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177227.258032:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177227.258033:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:0.0:1786177227.258036:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177227.258037:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.258039:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177227.258042:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856352768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.258048:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177227.258049:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177227.258051:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:0.0:1786177227.258053:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177227.258054:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177227.258055:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177227.258057:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177227.258058:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177227.258060:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177227.258061:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177227.258063:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:0.0:1786177227.258065:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177227.258066:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.258068:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177227.258070:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c380 x1872934856352896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.258076:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177227.258077:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177227.258078:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6800.
02000000:00000001:0.0:1786177227.258080:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177227.258081:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177227.258083:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177227.258084:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177227.258086:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786177227.258088:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177227.258089:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177227.258090:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c380.
00000100:00000001:0.0:1786177227.258092:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177227.258093:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.258095:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177227.258098:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856353024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177252 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177227.258104:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177227.258105:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177227.258106:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7000.
02000000:00000001:0.0:1786177227.258108:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177227.258109:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177227.258111:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177227.258112:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177227.258114:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786177227.258116:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177227.258117:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177227.258118:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:0.0:1786177227.258120:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177227.258121:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.258123:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177227.258126:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177227.258127:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177227.258129:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177227.258130:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177227.832606:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177227.832612:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177227.832618:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177227.832619:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177227.832623:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177227.832624:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177227.832683:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177227.832687:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177227.832689:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177228.280632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177228.280638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177228.280640:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177228.280646:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177228.280650:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177228.280653:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177228.280656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177228.280658:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177228.280661:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177228.856607:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177228.856612:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177228.856616:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177228.856617:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177228.856620:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177228.856621:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177228.856681:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177228.856685:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177228.856688:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177229.304669:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177229.304677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177229.304679:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177229.304686:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177229.304698:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177229.304701:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177229.304705:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177229.304708:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177229.304712:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177229.880619:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177229.880626:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177229.880632:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177229.880633:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177229.880637:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177229.880638:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177229.880689:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177229.880693:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177229.880696:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177230.328665:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177230.328673:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177230.328676:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177230.328682:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177230.328686:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177230.328689:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177230.328692:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177230.328695:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177230.328698:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177230.904607:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177230.904614:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177230.904616:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177230.904618:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177230.904622:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177230.904624:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177230.904625:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177230.904627:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177230.904630:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177230.904631:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177230.904632:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177230.904633:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177230.904635:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177230.904635:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177230.904637:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177230.904638:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177230.904640:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177230.904641:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177230.904695:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177230.904699:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177230.904702:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177231.096586:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302126797, at 4302126848
00000100:00000040:0.0:1786177231.096588:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302126797, at 4302126848
00000100:00000010:0.0:1786177231.096599:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000010:1.0:1786177231.096603:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134420.
00000100:00000001:0.0:1786177231.352645:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177231.352652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177231.352654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177231.352660:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177231.352664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177231.352667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177231.352670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177231.352673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177231.352676:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177231.928610:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177231.928616:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177231.928620:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177231.928622:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177231.928624:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177231.928625:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177231.928684:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177231.928687:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177231.928690:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786177232.376644:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177232.376654:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177232.376656:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177232.376659:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177232.376663:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177232.376670:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7455
00000400:00000200:0.0:1786177232.376679:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177232.376684:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177232.376687:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177232.376690:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.376693:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177232.376695:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.376697:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177232.376700:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177232.376702:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177232.376704:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177232.376707:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177232.376708:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177232.376710:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177232.376713:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.376714:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177232.376716:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.376717:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177232.376719:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177232.376722:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177232.376724:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177232.376726:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177232.376729:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177232.376731:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.376733:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177232.376735:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177232.376735:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177232.376740:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177232.376742:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177232.376745:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177232.376747:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000010:0.0:1786177232.376749:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41bb80.
00000100:00000001:2.0:1786177232.376751:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1786177232.376753:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
00000100:00000001:2.0:1786177232.376755:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1786177232.376758:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1786177232.376759:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177232.376761:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177232.376763:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.376765:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177232.376768:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177232.376772:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177232.376778:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000001:0.0:1786177232.376783:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177232.376791:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177232.376799:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177232.376802:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177232.376805:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9983a05] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177232.376820:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177232.376822:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177232.376825:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177232.376826:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177232.376827:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177232.376829:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177232.376831:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7455
00000400:00000200:0.0:1786177232.376834:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177232.376837:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177232.376838:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177232.376840:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.376842:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177232.376843:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1786177232.376845:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.376847:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177232.376853:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177232.376854:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1786177232.376856:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:3.0:1786177232.376857:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1786177232.376859:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177232.376862:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177232.376865:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177232.376866:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177232.376867:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177232.376875:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.376879:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177232.376882:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.376886:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177232.376889:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:3.0:1786177232.376889:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1786177232.376894:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177232.376894:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.376900:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177232.376902:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177232.376902:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177232.376907:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177232.376908:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177232.376913:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177232.376915:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.376916:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177232.376920:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00100000:3.0:1786177232.376922:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856353152:192.168.202.110@tcp:250:kworker.0
00000008:00000001:0.0:1786177232.376924:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.376928:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.376930:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:0.0:1786177232.376932:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419c00.
02000000:00000001:3.0:1786177232.376934:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:0.0:1786177232.376936:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177232.376937:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177232.376939:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177232.376941:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1786177232.376942:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.376944:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177232.376945:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177232.376947:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177232.376948:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000010:0.0:1786177232.376951:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:3.0:1786177232.376952:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.376957:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177232.376961:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:0.0:1786177232.376965:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1786177232.376968:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000001:0.0:1786177232.376975:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000010:3.0:1786177232.376976:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177232.376978:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177232.376982:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cf890c1c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177232.376984:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856353152, portal 25
00000100:00000001:3.0:1786177232.376989:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177232.376992:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761670016 : -114547947881600 : ffff97d1ba41bb80)
00000100:00000001:0.0:1786177232.377001:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177232.377002:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177232.377007:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177232.377013:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:0.0:1786177232.377015:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000040:3.0:1786177232.377018:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177232.377020:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:3.0:1786177232.377022:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00080000:0.0:1786177232.377024:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1786177232.377026:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856353152, offset 0
00000100:00000001:0.0:1786177232.377030:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177232.377034:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177232.377038:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7455
00000400:00000200:0.0:1786177232.377043:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177232.377047:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177232.377050:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177232.377052:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1786177232.377055:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:0.0:1786177232.377058:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.377061:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177232.377063:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1786177232.377065:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.377068:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177232.377069:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:0.0:1786177232.377072:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177232.377075:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000200:3.0:1786177232.377075:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:0.0:1786177232.377077:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000200:3.0:1786177232.377080:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000040:0.0:1786177232.377081:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177232.377085:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177232.377086:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254930:254930:256:4294967295] 192.168.202.110@tcp LPNI seq info [254919:254919:8:4294967295]
00000020:00000001:0.0:1786177232.377089:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177232.377094:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177232.377097:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.377101:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177232.377109:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177232.377110:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177232.377116:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.0:1786177232.377120:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:0.0:1786177232.377123:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000800:00000010:3.0:1786177232.377125:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177232.377132:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177232.377135:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000100:00080000:0.0:1786177232.377140:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:3.1:1786177232.377144:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177232.377145:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177232.377150:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:3.1:1786177232.377150:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
02000000:00000001:0.0:1786177232.377153:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000100:3.0:1786177232.377158:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786177232.377159:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.377163:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000800:00000010:3.0:1786177232.377163:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000008:00100000:0.0:1786177232.377167:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177232.377170:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177232.377170:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1786177232.377175:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177232.377180:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000010:0.0:1786177232.377181:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a300.
00000400:00000200:3.0:1786177232.377185:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000020:00000040:0.0:1786177232.377186:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000010:3.0:1786177232.377190:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
02000000:00000001:0.0:1786177232.377191:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177232.377194:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.377196:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177232.377196:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:0.0:1786177232.377198:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177232.377200:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177232.377202:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786177232.377205:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000100:00000001:0.0:1786177232.377210:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.377214:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00100000:0.0:1786177232.377218:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1786177232.377222:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177232.377231:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177232.377235:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177232.377242:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b0d3569] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.377244:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.377250:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177232.377254:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177232.377257:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177232.377259:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177232.377260:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177232.377262:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177232.377265:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177232.377268:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177232.377269:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177232.377272:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7455
00000100:00000001:3.0:1786177232.377274:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302131917
00000400:00000200:0.0:1786177232.377279:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:3.0:1786177232.377282:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad371e0.
00000400:00000200:0.0:1786177232.377283:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177232.377288:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177232.377290:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177232.377291:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786177232.377293:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177232.377296:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177232.377297:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302131917
00000100:00000001:0.0:1786177232.377299:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177232.377303:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000010:3.0:1786177232.377303:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37d20.
00000020:00000040:0.0:1786177232.377304:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:3.0:1786177232.377308:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177232.377311:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177232.377313:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177232.377315:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177232.377316:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000020:00000040:0.0:1786177232.377318:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177232.377321:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177232.377324:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00100000:3.0:1786177232.377324:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177232/real 1786177232]  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177232.377332:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.377336:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177232.377340:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.377343:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000001:3.0:1786177232.377344:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:0.0:1786177232.377345:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000400:00000001:3.0:1786177232.377347:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:0.0:1786177232.377350:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000010:3.0:1786177232.377351:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00000001:0.0:1786177232.377353:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177232.377355:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
02000000:00000001:0.0:1786177232.377356:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1786177232.377358:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
02000000:00000001:0.0:1786177232.377359:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177232.377361:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:0.0:1786177232.377363:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.377366:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000200:3.0:1786177232.377367:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1786177232.377369:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177232.377375:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.377378:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177232.377384:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a680.
00000100:00000200:3.0:1786177232.377385:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177232.377387:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177232.377391:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177232.377393:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.377396:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177232.377399:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177232.377401:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177232.377406:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:3.0:1786177232.377406:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177232.377411:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177232.377412:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.377418:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00100000:0.0:1786177232.377419:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177232.377439:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177232.377448:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.377478:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177232.377483:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177232.377486:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177232.377491:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.377491:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177232.377496:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.377498:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177232.377499:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177232.377502:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000911ee1e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.377502:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177232.377505:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177232.377529:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177232.377532:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177232.377534:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177232.377539:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177232.377546:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177232.377551:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1786177232.377553:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856353152:192.168.202.110@tcp:250:kworker.0
00000100:00080000:0.0:1786177232.377559:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177232.377561:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177232.377564:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177232.377564:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1786177232.377569:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:3.0:1786177232.377570:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41bb80 x1872934856353152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177232.377577:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177232.377579:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177232.377583:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00080000:0.0:1786177232.377584:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7455
02000000:00000010:3.0:1786177232.377586:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177232.377591:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000400:00000200:0.0:1786177232.377594:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1786177232.377595:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177232.377597:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177232.377600:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177232.377602:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
00000400:00000200:0.0:1786177232.377603:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:3.0:1786177232.377606:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00080000:0.0:1786177232.377608:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
02000000:00000001:3.0:1786177232.377609:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177232.377612:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41bb80.
00000100:00000040:0.0:1786177232.377615:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177232.377616:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177232.377618:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177232.377619:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177232.377623:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177232.377624:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177232.377626:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177232.377629:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177232.377630:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177232.377631:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177232.377632:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000020:00000040:0.0:1786177232.377633:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:3.0:1786177232.377634:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177232.377637:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:3.0:1786177232.377638:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1786177232.377641:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:3.0:1786177232.377642:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177232.377646:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177232.377647:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.377650:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177232.377652:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177232.377654:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177232.377658:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177232.377660:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177232.377662:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177232.377664:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177232.377667:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177232.377671:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177232.377672:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1786177232.377674:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.377675:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.377677:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:3.0:1786177232.377679:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:0.0:1786177232.377680:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000001:3.0:1786177232.377681:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:0.0:1786177232.377687:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177232.377687:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177232.377694:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177232.377695:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856353280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177232.377699:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:0.0:1786177232.377700:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419500.
02000000:00000001:3.0:1786177232.377701:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177232.377703:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177232.377706:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177232.377706:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177232.377711:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786177232.377712:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177232.377714:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177232.377717:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177232.377721:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177232.377723:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1786177232.377724:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
02000000:00000001:0.0:1786177232.377726:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1786177232.377729:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
02000000:00000001:0.0:1786177232.377731:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177232.377733:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856353280, portal 10
00000100:00000001:3.0:1786177232.377736:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177232.377737:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761661952 : -114547947889664 : ffff97d1ba419c00)
02000000:00000010:0.0:1786177232.377738:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000100:00000001:0.0:1786177232.377744:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177232.377745:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.377754:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:0.0:1786177232.377755:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1786177232.377758:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177232.377763:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177232.377767:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856353280, offset 0
00000400:00000200:3.0:1786177232.377774:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177232.377775:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177232.377779:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177232.377780:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177232.377784:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177232.377788:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1786177232.377790:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b6718a9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177232.377792:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177232.377797:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177232.377801:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177232.377808:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254931:254931:256:4294967295] 192.168.202.110@tcp LPNI seq info [254920:254920:8:4294967295]
00000400:00000200:3.0:1786177232.377816:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177232.377817:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177232.377823:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7465)
00000400:00000200:3.0:1786177232.377824:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177232.377829:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177232.377832:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177232.377835:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177232.377837:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786177232.377841:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177232.377843:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1786177232.377846:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177232.377847:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177232.377850:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177232.377854:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177232.377857:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177232.377858:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177232.377860:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177232.377863:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.377874:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177232.377877:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.377882:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.377884:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177232.377886:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177232.377889:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177232.377892:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177232.377893:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302128138
00000100:00000001:3.0:1786177232.377896:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302131917
00000100:00000001:3.0:1786177232.377898:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302128138
00000100:00000001:3.0:1786177232.377900:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302131917
00000100:00000001:3.0:1786177232.377901:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177232.377902:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.377903:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177232.377906:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177232/real 1786177232]  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177232.377912:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177232.377913:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177232.377914:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177232.377916:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177232.377917:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177232.377919:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177232.377921:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177232.377926:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.377930:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177232.377932:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.377933:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177232.377934:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177232.377936:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.377940:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177232.377943:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.377947:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177232.377948:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.377949:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177232.377950:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.377951:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177232.377952:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177232.377954:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177232.377956:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177232.377957:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177232.377960:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177232.377964:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856353280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177232.377967:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177232.377968:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.377973:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177232.377974:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177232.377976:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177232.377977:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177232.377978:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177232.377980:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856353408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177232.377982:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177232.377983:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177232.377984:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177232.377986:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177232.377988:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177232.377989:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177232.377991:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177232.377992:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177232.377994:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177232.377996:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856353408, portal 10
00000100:00000001:3.0:1786177232.377997:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177232.377998:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761663744 : -114547947887872 : ffff97d1ba41a300)
00000100:00000040:3.0:1786177232.378002:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378006:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177232.378007:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177232.378014:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177232.378015:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856353408, offset 0
00000400:00000200:3.0:1786177232.378018:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177232.378020:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177232.378023:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177232.378025:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177232.378026:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177232.378028:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177232.378029:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177232.378032:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254932:254932:256:4294967295] 192.168.202.110@tcp LPNI seq info [254921:254921:8:4294967295]
00000400:00000200:3.0:1786177232.378034:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177232.378039:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177232.378042:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177232.378043:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177232.378046:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177232.378048:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786177232.378050:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177232.378052:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1786177232.378055:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177232.378056:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177232.378058:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177232.378060:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177232.378062:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177232.378063:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177232.378065:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177232.378067:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378072:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177232.378074:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378078:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.378079:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177232.378081:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177232.378083:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177232.378085:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177232.378086:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302128263
00000100:00000001:3.0:1786177232.378088:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302131917
00000100:00000001:3.0:1786177232.378090:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302128263
00000100:00000001:3.0:1786177232.378091:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302131917
00000100:00000001:3.0:1786177232.378092:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177232.378094:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.378095:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177232.378097:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177232/real 1786177232]  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177232.378101:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177232.378103:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177232.378104:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177232.378105:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177232.378106:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177232.378107:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177232.378109:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177232.378114:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378118:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177232.378119:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378120:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177232.378121:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177232.378123:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378127:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177232.378129:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378133:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177232.378135:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378136:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177232.378137:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378138:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177232.378139:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177232.378141:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177232.378143:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177232.378144:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177232.378146:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177232.378155:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856353408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177232.378158:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177232.378160:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378164:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177232.378165:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177232.378166:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177232.378167:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177232.378168:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177232.378170:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856353536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177232.378172:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177232.378173:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177232.378173:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177232.378175:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177232.378179:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177232.378180:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177232.378182:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177232.378184:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177232.378186:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177232.378188:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856353536, portal 4
00000100:00000001:3.0:1786177232.378189:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177232.378190:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761664640 : -114547947886976 : ffff97d1ba41a680)
00000100:00000040:3.0:1786177232.378194:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378198:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177232.378199:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177232.378201:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177232.378202:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856353536, offset 0
00000400:00000200:3.0:1786177232.378204:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177232.378207:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177232.378209:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177232.378211:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177232.378213:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177232.378214:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177232.378216:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177232.378218:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254933:254933:256:4294967295] 192.168.202.110@tcp LPNI seq info [254922:254922:8:4294967295]
00000400:00000200:3.0:1786177232.378220:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177232.378224:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177232.378226:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177232.378228:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177232.378230:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177232.378232:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786177232.378235:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177232.378237:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1786177232.378239:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177232.378241:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177232.378242:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177232.378244:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177232.378246:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177232.378247:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177232.378249:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177232.378251:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378256:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177232.378258:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378262:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.378263:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177232.378264:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177232.378266:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177232.378268:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177232.378270:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302128138
00000100:00000001:3.0:1786177232.378271:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302131917
00000100:00000001:3.0:1786177232.378273:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302128138
00000100:00000001:3.0:1786177232.378274:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302131917
00000100:00000001:3.0:1786177232.378275:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177232.378276:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.378277:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177232.378279:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177232/real 1786177232]  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177232.378284:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177232.378285:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177232.378285:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177232.378287:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177232.378288:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177232.378289:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177232.378291:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177232.378296:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378300:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177232.378301:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378302:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177232.378303:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177232.378305:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378309:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177232.378311:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378315:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177232.378317:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378318:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177232.378319:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378320:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177232.378321:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177232.378322:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177232.378324:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177232.378325:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177232.378328:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177232.378332:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856353536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177232.378334:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378335:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177232.378337:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419c00 x1872934856353280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378341:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177232.378342:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177232.378343:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177232.378344:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177232.378345:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177232.378346:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177232.378347:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177232.378348:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:3.0:1786177232.378350:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177232.378351:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177232.378352:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419c00.
00000100:00000001:3.0:1786177232.378354:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177232.378355:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378356:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177232.378358:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a300 x1872934856353408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378367:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177232.378367:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177232.378368:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177232.378370:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177232.378371:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177232.378372:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177232.378373:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177232.378374:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:3.0:1786177232.378375:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177232.378376:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177232.378377:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a300.
00000100:00000001:3.0:1786177232.378378:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177232.378379:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378380:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177232.378381:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a680 x1872934856353536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378385:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177232.378386:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177232.378386:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177232.378388:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177232.378389:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177232.378390:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177232.378391:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177232.378393:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:3.0:1786177232.378395:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177232.378396:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177232.378397:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a680.
00000100:00000001:3.0:1786177232.378399:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177232.378400:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378402:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378405:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177232.378407:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.378409:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177232.378410:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177232.378411:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177232.378416:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378435:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177232.378437:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177232.378440:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177232.378441:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177232.378443:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177232.378446:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856353664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177232.378450:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177232.378452:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177232.378453:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177232.378455:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177232.378459:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177232.378460:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177232.378464:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177232.378466:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177232.378468:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177232.378470:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856353664, portal 4
00000100:00000001:3.0:1786177232.378471:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177232.378472:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761660160 : -114547947891456 : ffff97d1ba419500)
00000100:00000040:3.0:1786177232.378477:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378481:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177232.378483:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177232.378485:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177232.378486:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856353664, offset 0
00000400:00000200:3.0:1786177232.378489:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177232.378491:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177232.378494:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177232.378496:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177232.378497:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177232.378499:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177232.378501:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177232.378503:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254934:254934:256:4294967295] 192.168.202.110@tcp LPNI seq info [254923:254923:8:4294967295]
00000400:00000200:3.0:1786177232.378506:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177232.378525:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177232.378528:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177232.378531:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177232.378533:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177232.378536:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1786177232.378538:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177232.378540:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1786177232.378542:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177232.378544:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177232.378546:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177232.378548:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177232.378550:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177232.378551:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177232.378553:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177232.378555:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378561:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177232.378563:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378567:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.378568:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177232.378570:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177232.378572:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177232.378574:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177232.378576:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302128147
00000100:00000001:3.0:1786177232.378577:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302131917
00000100:00000001:3.0:1786177232.378579:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302128147
00000100:00000001:3.0:1786177232.378580:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302131917
00000100:00000001:3.0:1786177232.378582:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177232.378583:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.378585:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177232.378587:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177232/real 1786177232]  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177232.378592:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177232.378593:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177232.378594:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177232.378595:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177232.378596:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177232.378597:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177232.378599:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177232.378605:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378608:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177232.378609:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378610:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177232.378611:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177232.378613:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378617:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177232.378619:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378623:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177232.378625:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378626:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177232.378627:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378629:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177232.378630:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177232.378632:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177232.378635:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177232.378636:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177232.378639:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177232.378643:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856353664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177232.378645:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378647:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177232.378648:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419500 x1872934856353664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177232.378652:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177232.378653:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177232.378654:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177232.378655:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177232.378656:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177232.378657:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177232.378658:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177232.378659:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:3.0:1786177232.378661:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177232.378661:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177232.378662:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419500.
00000100:00000001:3.0:1786177232.378664:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177232.378664:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378665:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177232.378668:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177232.378669:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177232.378670:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177232.378671:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177232.952667:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177232.952675:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177232.952680:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177232.952681:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177232.952683:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177232.952684:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177232.952745:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177232.952749:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177232.952752:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177233.400676:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177233.400686:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177233.400688:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177233.400696:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177233.400700:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177233.400705:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177233.400708:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177233.400713:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177233.400716:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177233.976679:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177233.976687:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177233.976692:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177233.976693:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177233.976695:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177233.976699:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177233.976701:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177233.976705:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177233.976708:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177233.976710:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177233.976712:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177233.976714:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177233.976782:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177233.976794:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177233.976798:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177234.424674:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177234.424683:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177234.424685:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177234.424691:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177234.424696:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177234.424699:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177234.424702:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177234.424710:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177234.424713:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177235.000618:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177235.000627:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177235.000633:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177235.000635:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177235.000638:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177235.000640:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177235.000699:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177235.000703:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177235.000706:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177235.448681:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177235.448693:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177235.448695:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177235.448702:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177235.448707:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177235.448712:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177235.448715:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177235.448719:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177235.448722:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177236.024669:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177236.024677:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177236.024683:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177236.024684:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177236.024687:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177236.024687:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177236.024758:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177236.024762:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177236.024765:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177236.216649:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302131917, at 4302131968
00000100:00000040:1.0:1786177236.216649:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302131917, at 4302131968
00000100:00000010:0.0:1786177236.216673:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad371e0.
00000100:00000010:1.0:1786177236.216679:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37d20.
00000100:00000001:2.0:1786177236.472657:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177236.472667:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177236.472670:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177236.472675:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177236.472679:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177236.472683:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177236.472686:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177236.472689:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177236.472691:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177237.048614:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177237.048620:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177237.048625:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177237.048626:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177237.048628:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177237.048629:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177237.048686:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177237.048691:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177237.048695:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177237.496595:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177237.496603:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177237.496606:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177237.496613:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177237.496617:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177237.496620:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177237.496623:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177237.496625:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177237.496628:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177237.496645:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177237.496655:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177237.496658:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177237.496663:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177237.496667:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177237.496675:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7460
00000400:00000200:3.0:1786177237.496683:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177237.496688:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177237.496691:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177237.496694:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.496697:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177237.496699:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.496701:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177237.496705:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177237.496707:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177237.496710:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177237.496712:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177237.496715:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177237.496717:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177237.496719:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.496720:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177237.496722:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.496723:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177237.496725:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177237.496728:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177237.496729:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177237.496731:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177237.496734:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177237.496736:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177237.496737:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177237.496739:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177237.496747:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69fb80.
00000020:00000040:3.0:1786177237.496749:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177237.496752:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177237.496753:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177237.496756:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177237.496758:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177237.496762:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177237.496768:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:3.0:1786177237.496773:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177237.496780:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177237.496788:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177237.496792:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177237.496795:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014f55bbb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177237.496817:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177237.496820:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177237.496822:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177237.496823:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177237.496824:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177237.496827:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177237.496829:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7460
00000400:00000200:3.0:1786177237.496832:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177237.496836:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177237.496839:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177237.496842:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.496844:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177237.496846:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1786177237.496846:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177237.496851:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177237.496852:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177237.496856:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1786177237.496857:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177237.496868:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177237.496872:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177237.496874:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786177237.496878:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177237.496879:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177237.496882:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177237.496898:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.496906:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177237.496913:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:3.0:1786177237.496917:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177237.496918:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177237.496927:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:0.0:1786177237.496930:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:3.0:1786177237.496930:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
02000000:00000001:0.0:1786177237.496937:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1786177237.496938:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1786177237.496955:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177237.496955:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177237.496970:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:0.0:1786177237.496975:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856353792:192.168.202.110@tcp:250:kworker.0
02000000:00000001:3.0:1786177237.496977:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177237.496987:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.496988:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1786177237.496996:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:0.0:1786177237.496999:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:3.0:1786177237.497000:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1786177237.497003:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1786177237.497006:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177237.497008:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1786177237.497009:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177237.497014:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ca80.
02000000:00000010:0.0:1786177237.497017:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
00000020:00000040:3.0:1786177237.497018:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177237.497029:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177237.497030:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177237.497033:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177237.497036:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177237.497038:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:0.0:1786177237.497044:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786177237.497045:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177237.497052:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
02000000:00000010:3.0:1786177237.497053:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000001:3.0:1786177237.497061:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177237.497064:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00100000:3.0:1786177237.497072:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1786177237.497077:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856353792, portal 25
00000100:00000001:3.0:1786177237.497089:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177237.497090:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177237.497095:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000001:3.0:1786177237.497095:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177237.497116:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177237.497117:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ffb0865] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.497156:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177237.497164:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177237.497164:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177237.497171:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1786177237.497172:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177237.497182:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856353792, offset 0
00000100:00080000:3.0:1786177237.497182:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177237.497185:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177237.497188:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177237.497191:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177237.497195:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7460
00000400:00000200:0.0:1786177237.497204:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177237.497205:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177237.497214:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177237.497216:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177237.497219:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.497221:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177237.497224:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177237.497229:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177237.497229:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177237.497235:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177237.497237:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177237.497239:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177237.497241:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786177237.497242:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177237.497244:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:0.0:1786177237.497248:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:3.0:1786177237.497250:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.497259:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177237.497264:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.497266:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786177237.497269:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:3.0:1786177237.497270:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786177237.497285:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:0.0:1786177237.497288:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1786177237.497294:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177237.497297:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786177237.497298:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1786177237.497302:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1786177237.497303:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:3.0:1786177237.497310:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177237.497320:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254935:254935:256:4294967295] 192.168.202.110@tcp LPNI seq info [254924:254924:8:4294967295]
00000100:00000001:3.0:1786177237.497321:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177237.497328:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1786177237.497339:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:3.0:1786177237.497340:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177237.497344:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177237.497348:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f800.
00000020:00000040:3.0:1786177237.497355:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:0.0:1786177237.497364:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:3.0:1786177237.497366:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177237.497385:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177237.497394:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177237.497394:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177237.497403:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000010:0.0:1786177237.497405:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
02000000:00000001:3.0:1786177237.497406:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177237.497414:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000800:00000200:0.0:1786177237.497420:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177237.497420:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.0:1786177237.497436:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000100:00100000:3.0:1786177237.497439:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:0.1:1786177237.497451:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177237.497460:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000100:00000001:3.0:1786177237.497461:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177237.497467:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:0.0:1786177237.497468:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177237.497477:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000100:00000040:3.0:1786177237.497478:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4d339d9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177237.497489:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177237.497491:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1786177237.497493:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:3.0:1786177237.497495:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177237.497498:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177237.497503:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177237.497508:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1786177237.497545:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177237.497553:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177237.497557:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177237.497566:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:3.0:1786177237.497566:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7460
00000100:00000200:0.0:1786177237.497575:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177237.497577:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177237.497587:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177237.497593:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177237.497603:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.497608:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177237.497609:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177237.497613:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:0.0:1786177237.497617:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177237.497619:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177237.497628:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177237.497632:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177237.497637:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786177237.497639:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177237.497641:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:0.0:1786177237.497644:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:3.0:1786177237.497645:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177237.497650:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:0.0:1786177237.497654:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177237.497654:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.497663:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177237.497668:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177237.497671:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177237.497675:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177237.497677:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302137038
00000020:00000040:3.0:1786177237.497678:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000010:0.0:1786177237.497691:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00080000:3.0:1786177237.497691:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177237.497694:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177237.497697:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177237.497703:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:3.0:1786177237.497704:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177237.497715:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.497719:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302137038
00000100:00000001:3.0:1786177237.497719:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177237.497725:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177237.497729:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177237.497730:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134d20.
00000100:00000001:3.0:1786177237.497731:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177237.497741:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f480.
00000100:00000001:0.0:1786177237.497742:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000020:00000040:3.0:1786177237.497744:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177237.497749:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177237.497750:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.497751:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177237.497753:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177237.497756:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1786177237.497762:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177237.497762:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177237.497774:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:0.0:1786177237.497775:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1786177237.497778:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177237.497792:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1786177237.497793:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177237/real 1786177237]  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177237.497819:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177237.497828:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177237.497835:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092aa1aa7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177237.497840:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1786177237.497842:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:0.0:1786177237.497846:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:3.0:1786177237.497846:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177237.497854:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000010:0.0:1786177237.497855:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000100:00000001:3.0:1786177237.497856:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177237.497860:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1786177237.497865:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000100:00000001:3.0:1786177237.497865:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177237.497872:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7460
00000400:00000010:0.0:1786177237.497873:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000400:00000200:3.0:1786177237.497880:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177237.497887:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000400:00000200:3.0:1786177237.497888:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177237.497894:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177237.497904:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:0.0:1786177237.497905:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177237.497908:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177237.497912:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.497918:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177237.497929:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177237.497933:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177237.497943:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177237.497946:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000200:0.0:1786177237.497948:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177237.497951:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177237.497960:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:0.0:1786177237.497973:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:3.0:1786177237.497976:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177237.497978:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.497987:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177237.497988:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177237.497996:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177237.497999:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177237.498003:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00100000:0.0:1786177237.498009:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177237.498010:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786177237.498023:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177237.498035:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177237.498045:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1786177237.498050:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177237.498051:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177237.498062:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177237.498066:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:0.0:1786177237.498068:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1786177237.498069:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177237.498077:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177237.498086:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177237.498093:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f100.
00000100:00000001:0.0:1786177237.498101:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:3.0:1786177237.498103:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:0.0:1786177237.498113:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177237.498114:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1786177237.498121:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1786177237.498122:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.498123:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.498132:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177237.498134:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786177237.498135:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177237.498140:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:3.0:1786177237.498140:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177237.498145:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177237.498147:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177237.498151:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000010:3.0:1786177237.498156:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000040:0.0:1786177237.498157:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177237.498170:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177237.498182:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:0.0:1786177237.498184:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856353792:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177237.498206:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177237.498207:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177237.498210:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.498215:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177237.498219:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1ef9f45] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177237.498227:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856353792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177237.498231:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177237.498239:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7470)
00000100:00000001:0.0:1786177237.498250:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177237.498254:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177237.498259:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786177237.498263:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177237.498267:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177237.498269:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177237.498272:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177237.498277:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786177237.498283:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177237.498285:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177237.498287:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:0.0:1786177237.498291:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177237.498292:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.498295:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.498303:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177237.498305:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.498307:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177237.498309:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177237.498311:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177237.498315:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498328:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177237.498330:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177237.498332:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177237.498334:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177237.498337:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177237.498340:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856353920:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177237.498343:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177237.498346:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177237.498348:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177237.498350:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177237.498353:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786177237.498355:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177237.498360:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177237.498363:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177237.498368:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177237.498383:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856353920, portal 10
00000100:00000001:0.0:1786177237.498387:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177237.498388:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:0.0:1786177237.498396:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498401:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177237.498403:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177237.498405:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177237.498408:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856353920, offset 0
00000400:00000200:0.0:1786177237.498411:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177237.498415:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177237.498419:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177237.498421:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177237.498423:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177237.498425:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177237.498427:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177237.498430:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254936:254936:256:4294967295] 192.168.202.110@tcp LPNI seq info [254925:254925:8:4294967295]
00000400:00000200:0.0:1786177237.498433:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177237.498438:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177237.498442:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177237.498445:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177237.498447:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177237.498450:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786177237.498453:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177237.498455:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1786177237.498458:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177237.498460:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177237.498462:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177237.498465:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177237.498467:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177237.498469:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177237.498470:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177237.498473:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498479:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177237.498481:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498485:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.498486:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177237.498488:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177237.498491:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177237.498493:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177237.498494:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302133258
00000100:00000001:0.0:1786177237.498497:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302137038
00000100:00000001:0.0:1786177237.498498:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302133258
00000100:00000001:0.0:1786177237.498500:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302137038
00000100:00000001:0.0:1786177237.498502:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177237.498504:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.498505:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177237.498508:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177237/real 1786177237]  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177237.498534:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177237.498535:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177237.498536:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177237.498538:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177237.498539:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177237.498541:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177237.498543:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177237.498549:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498553:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177237.498554:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.498555:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177237.498556:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177237.498558:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498563:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177237.498566:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498571:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177237.498572:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.498573:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177237.498574:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.498575:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177237.498577:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177237.498578:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177237.498581:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177237.498583:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177237.498586:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177237.498591:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856353920:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177237.498593:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177237.498596:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498600:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177237.498601:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177237.498602:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177237.498604:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177237.498605:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177237.498607:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177237.498610:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177237.498611:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177237.498613:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177237.498614:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177237.498616:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4800.
02000000:00000001:0.0:1786177237.498617:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177237.498619:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177237.498621:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177237.498623:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177237.498625:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856354048, portal 10
00000100:00000001:0.0:1786177237.498627:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177237.498628:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
00000100:00000040:0.0:1786177237.498633:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498637:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177237.498639:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177237.498640:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177237.498642:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856354048, offset 0
00000400:00000200:0.0:1786177237.498645:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177237.498648:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177237.498650:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177237.498653:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177237.498654:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177237.498656:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177237.498658:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177237.498660:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254937:254937:256:4294967295] 192.168.202.110@tcp LPNI seq info [254926:254926:8:4294967295]
00000400:00000200:0.0:1786177237.498663:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177237.498667:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177237.498670:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177237.498672:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177237.498675:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177237.498677:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786177237.498679:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177237.498681:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1786177237.498684:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177237.498686:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177237.498688:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177237.498690:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177237.498692:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177237.498693:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177237.498695:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177237.498697:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498703:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177237.498705:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498709:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.498710:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177237.498712:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177237.498714:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177237.498717:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177237.498718:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302133384
00000100:00000001:0.0:1786177237.498720:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302137038
00000100:00000001:0.0:1786177237.498722:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302133384
00000100:00000001:0.0:1786177237.498723:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302137038
00000100:00000001:0.0:1786177237.498724:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177237.498725:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.498727:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177237.498729:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177237/real 1786177237]  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177237.498733:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177237.498734:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177237.498735:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177237.498737:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177237.498738:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177237.498739:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177237.498742:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177237.498747:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498750:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177237.498751:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.498752:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177237.498753:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177237.498755:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498759:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177237.498762:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498767:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177237.498769:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.498770:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177237.498771:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.498772:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177237.498773:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177237.498774:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177237.498776:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177237.498778:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177237.498780:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177237.498784:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177237.498787:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177237.498789:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498794:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177237.498795:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177237.498796:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177237.498798:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177237.498799:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177237.498800:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177237.498802:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177237.498803:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177237.498804:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177237.498805:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177237.498807:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177237.498808:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177237.498811:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177237.498812:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177237.498815:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177237.498817:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856354176, portal 4
00000100:00000001:0.0:1786177237.498818:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177237.498819:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:0.0:1786177237.498824:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498828:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177237.498829:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177237.498831:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177237.498832:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856354176, offset 0
00000400:00000200:0.0:1786177237.498835:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177237.498837:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177237.498839:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177237.498841:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177237.498843:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177237.498845:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177237.498846:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177237.498848:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254938:254938:256:4294967295] 192.168.202.110@tcp LPNI seq info [254927:254927:8:4294967295]
00000400:00000200:0.0:1786177237.498851:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177237.498856:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177237.498858:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177237.498860:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177237.498862:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177237.498864:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786177237.498866:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177237.498868:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1786177237.498870:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177237.498872:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177237.498874:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177237.498876:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177237.498878:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177237.498880:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177237.498881:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177237.498884:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498889:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177237.498891:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498895:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.498896:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177237.498898:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177237.498900:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177237.498902:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177237.498903:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302133259
00000100:00000001:0.0:1786177237.498905:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302137038
00000100:00000001:0.0:1786177237.498907:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302133259
00000100:00000001:0.0:1786177237.498908:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302137038
00000100:00000001:0.0:1786177237.498910:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177237.498911:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.498912:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177237.498914:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177237/real 1786177237]  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177237.498918:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177237.498919:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177237.498920:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177237.498921:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177237.498922:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177237.498923:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177237.498925:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177237.498932:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498936:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177237.498938:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.498939:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177237.498940:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177237.498942:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498946:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177237.498948:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498952:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177237.498954:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.498954:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177237.498955:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.498956:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177237.498957:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177237.498959:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177237.498961:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177237.498962:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177237.498964:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177237.498969:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177237.498971:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177237.498973:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.498977:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177237.498978:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177237.498980:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177237.498981:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177237.498982:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177237.498984:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177237.498986:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177237.498988:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177237.498988:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177237.498989:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177237.498993:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177237.498994:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177237.498996:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177237.498998:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177237.498999:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177237.499001:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856354304, portal 4
00000100:00000001:0.0:1786177237.499003:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177237.499004:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:0.0:1786177237.499008:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.499012:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177237.499013:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177237.499015:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177237.499017:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856354304, offset 0
00000400:00000200:0.0:1786177237.499019:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177237.499021:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177237.499022:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177237.499024:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177237.499025:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177237.499027:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177237.499028:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177237.499030:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254939:254939:256:4294967295] 192.168.202.110@tcp LPNI seq info [254928:254928:8:4294967295]
00000400:00000200:0.0:1786177237.499033:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177237.499037:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177237.499041:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177237.499042:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177237.499045:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177237.499047:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1786177237.499049:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177237.499051:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1786177237.499054:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177237.499055:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177237.499056:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177237.499058:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177237.499060:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177237.499061:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177237.499063:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177237.499065:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.499070:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177237.499072:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.499075:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.499077:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177237.499078:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177237.499081:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177237.499083:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177237.499084:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302133267
00000100:00000001:0.0:1786177237.499086:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302137038
00000100:00000001:0.0:1786177237.499088:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302133267
00000100:00000001:0.0:1786177237.499089:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302137038
00000100:00000001:0.0:1786177237.499090:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177237.499091:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.499092:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177237.499094:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177237/real 1786177237]  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177237.499100:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177237.499101:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177237.499102:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177237.499104:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177237.499105:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177237.499106:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177237.499108:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177237.499112:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.499116:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177237.499117:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.499127:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177237.499128:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177237.499131:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.499137:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177237.499140:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.499144:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177237.499145:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.499146:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177237.499147:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.499149:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177237.499150:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177237.499152:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177237.499154:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177237.499155:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177237.499158:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177237.499164:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177237.499166:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.499167:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177237.499169:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856353920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.499173:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177237.499174:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177237.499176:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:0.0:1786177237.499177:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177237.499178:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177237.499180:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177237.499181:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177237.499182:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786177237.499185:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177237.499186:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177237.499187:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:0.0:1786177237.499188:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177237.499189:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.499191:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177237.499193:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856354048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.499197:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177237.499198:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177237.499199:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4800.
02000000:00000001:0.0:1786177237.499200:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177237.499201:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177237.499202:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177237.499203:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177237.499204:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786177237.499205:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177237.499206:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177237.499207:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000001:0.0:1786177237.499209:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177237.499210:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.499211:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177237.499213:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856354176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.499217:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177237.499218:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177237.499218:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:0.0:1786177237.499220:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177237.499220:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177237.499221:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177237.499222:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177237.499223:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786177237.499224:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177237.499225:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177237.499225:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:0.0:1786177237.499226:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177237.499227:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.499228:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177237.499230:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856354304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177237.499234:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177237.499234:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177237.499235:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177237.499236:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177237.499237:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177237.499238:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177237.499239:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177237.499240:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177237.499241:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177237.499242:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177237.499243:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:0.0:1786177237.499244:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177237.499244:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.499246:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177237.499248:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177237.499249:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177237.499251:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177237.499252:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177238.072605:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177238.072611:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177238.072616:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177238.072617:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177238.072619:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177238.072620:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177238.072684:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177238.072689:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177238.072693:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177238.520672:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177238.520684:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177238.520687:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177238.520695:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177238.520700:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177238.520705:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177238.520709:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177238.520713:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177238.520717:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177239.096629:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177239.096637:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177239.096644:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177239.096645:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177239.096649:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177239.096650:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177239.096735:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177239.096738:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177239.096741:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177239.544597:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177239.544656:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177239.544660:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177239.544665:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177239.544669:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177239.544673:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177239.544675:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177239.544678:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177239.544682:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177240.120607:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177240.120612:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177240.120616:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177240.120617:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177240.120627:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177240.120628:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177240.120680:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177240.120683:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177240.120686:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177240.568591:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177240.568597:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177240.568598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177240.568603:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177240.568605:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177240.568608:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177240.568610:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177240.568612:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177240.568614:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177241.144635:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177241.144644:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177241.144647:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177241.144651:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177241.144656:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177241.144659:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177241.144661:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177241.144664:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177241.144668:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177241.144670:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177241.144672:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177241.144673:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177241.144675:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177241.144677:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177241.144679:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177241.144681:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177241.144685:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177241.144687:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177241.144739:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177241.144750:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177241.144752:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177241.336623:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302137038, at 4302137088
00000100:00000040:1.0:1786177241.336623:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302137038, at 4302137088
00000100:00000010:1.0:1786177241.336639:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134d20.
00000100:00000010:0.0:1786177241.336642:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:2.0:1786177241.592646:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177241.592658:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177241.592661:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177241.592669:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177241.592674:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177241.592678:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177241.592681:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177241.592683:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177241.592686:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177242.168616:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177242.168622:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177242.168626:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177242.168628:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177242.168629:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177242.168630:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177242.168688:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177242.168691:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177242.168694:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177242.616618:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177242.616625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177242.616627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177242.616632:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177242.616636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177242.616639:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177242.616641:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177242.616644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177242.616647:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177242.616651:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177242.616662:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177242.616665:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177242.616670:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177242.616676:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177242.616685:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7465
00000400:00000200:0.0:1786177242.616695:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177242.616703:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177242.616706:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177242.616710:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177242.616715:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177242.616718:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177242.616721:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177242.616726:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177242.616728:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177242.616732:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177242.616736:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177242.616738:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177242.616741:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177242.616744:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177242.616748:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177242.616750:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177242.616753:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177242.616756:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177242.616771:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177242.616774:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177242.616781:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177242.616785:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177242.616789:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177242.616792:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177242.616794:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177242.616806:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41aa00.
00000020:00000040:0.0:1786177242.616810:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177242.616815:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177242.616818:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177242.616822:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177242.616825:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177242.616830:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177242.616839:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
00000100:00000001:0.0:1786177242.616847:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177242.616859:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177242.616871:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177242.616876:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177242.616882:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da0255a4] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177242.616909:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177242.616913:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177242.616917:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177242.616919:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177242.616921:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177242.616924:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177242.616929:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7465
00000400:00000200:0.0:1786177242.616934:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177242.616939:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177242.616941:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177242.616944:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1786177242.616946:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177242.616949:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177242.616950:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177242.616953:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177242.616955:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177242.616959:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:3.0:1786177242.616963:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177242.616964:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177242.616967:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177242.616970:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177242.616974:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:3.0:1786177242.616977:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000001:0.0:1786177242.616979:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:3.0:1786177242.616980:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177242.616986:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:3.0:1786177242.616987:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177242.616991:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1786177242.616994:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177242.616995:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177242.616998:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:3.0:1786177242.617002:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856354432:192.168.202.110@tcp:250:kworker.0
00000100:00000040:0.0:1786177242.617006:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177242.617008:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1786177242.617010:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177242.617013:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:0.0:1786177242.617014:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
02000000:00000001:3.0:1786177242.617016:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177242.617020:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1786177242.617021:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1786177242.617026:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177242.617028:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786177242.617029:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177242.617033:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177242.617037:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177242.617040:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786177242.617043:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177242.617046:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000001:0.0:1786177242.617047:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177242.617053:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000010:3.0:1786177242.617054:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000008:00000001:0.0:1786177242.617059:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177242.617063:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177242.617065:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856354432, portal 25
00000100:00000010:0.0:1786177242.617068:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41ad80.
00000100:00000001:3.0:1786177242.617069:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177242.617072:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761665536 : -114547947886080 : ffff97d1ba41aa00)
00000020:00000040:0.0:1786177242.617074:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177242.617080:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:3.0:1786177242.617080:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177242.617084:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177242.617089:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177242.617092:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177242.617093:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1786177242.617096:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177242.617097:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177242.617103:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
02000000:00000010:0.0:1786177242.617105:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4c00.
00000100:00000200:3.0:1786177242.617106:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856354432, offset 0
00000100:00000001:0.0:1786177242.617114:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177242.617118:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00100000:0.0:1786177242.617125:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177242.617131:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177242.617140:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177242.617141:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1786177242.617145:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177242.617149:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1786177242.617153:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034e2c676] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177242.617157:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177242.617165:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177242.617170:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1786177242.617173:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177242.617175:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254940:254940:256:4294967295] 192.168.202.110@tcp LPNI seq info [254929:254929:8:4294967295]
00000100:00080000:0.0:1786177242.617178:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1786177242.617183:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177242.617186:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177242.617190:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177242.617194:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1786177242.617198:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1786177242.617200:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177242.617206:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7465
00000800:00000200:3.0:1786177242.617209:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177242.617214:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000010:3.0:1786177242.617217:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000400:00000200:0.0:1786177242.617221:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177242.617225:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1786177242.617228:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786177242.617233:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:3.0:1786177242.617235:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1786177242.617237:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177242.617242:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:3.1:1786177242.617246:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177242.617247:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.1:1786177242.617251:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:0.0:1786177242.617252:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177242.617256:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177242.617260:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000100:3.0:1786177242.617260:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177242.617264:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000020:00000040:0.0:1786177242.617265:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177242.617270:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177242.617273:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:0.0:1786177242.617274:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177242.617281:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177242.617282:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1786177242.617287:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177242.617290:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000100:00000040:0.0:1786177242.617293:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:3.0:1786177242.617294:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:0.0:1786177242.617298:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177242.617300:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:0.0:1786177242.617303:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000200:3.0:1786177242.617308:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177242.617310:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177242.617316:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177242.617321:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177242.617340:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177242.617343:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.617343:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177242.617347:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:3.0:1786177242.617349:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1786177242.617351:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177242.617358:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177242.617363:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.617364:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.617367:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000010:0.0:1786177242.617368:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b480.
00000020:00000040:0.0:1786177242.617373:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000001:3.0:1786177242.617376:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177242.617377:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177242.617380:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177242.617383:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1786177242.617384:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786177242.617387:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177242.617388:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302142157
02000000:00000001:0.0:1786177242.617391:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177242.617395:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37300.
02000000:00000010:0.0:1786177242.617398:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000100:00000001:0.0:1786177242.617403:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.617404:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00100000:0.0:1786177242.617411:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177242.617412:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302142157
00000100:00000010:3.0:1786177242.617419:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37600.
00000100:00000001:3.0:1786177242.617424:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:0.0:1786177242.617429:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177242.617430:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177242.617433:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.617435:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.617439:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:0.0:1786177242.617441:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001215e9e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177242.617446:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177242/real 1786177242]  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177242.617469:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177242.617471:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00080000:0.0:1786177242.617474:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:3.0:1786177242.617475:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177242.617478:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00080000:0.0:1786177242.617481:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:3.0:1786177242.617483:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000100:00000001:0.0:1786177242.617484:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:3.0:1786177242.617488:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00080000:0.0:1786177242.617489:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177242.617491:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1786177242.617494:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:3.0:1786177242.617495:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177242.617500:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7465
00000100:00000200:3.0:1786177242.617531:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.617537:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177242.617539:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.617542:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177242.617543:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177242.617547:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.617552:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177242.617559:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.617568:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177242.617571:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1786177242.617574:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177242.617574:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177242.617576:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.617582:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000400:00000200:0.0:1786177242.617583:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177242.617585:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177242.617587:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1786177242.617593:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:0.0:1786177242.617594:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177242.617600:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00080000:3.0:1786177242.617603:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000040:0.0:1786177242.617606:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177242.617610:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1786177242.617611:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177242.617618:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177242.617619:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177242.617622:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000040:3.0:1786177242.617623:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177242.617625:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177242.617631:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177242.617636:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177242.617644:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00100000:3.0:1786177242.617648:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856354432:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177242.617649:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177242.617654:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177242.617659:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177242.617660:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1786177242.617663:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:3.0:1786177242.617665:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1786177242.617668:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177242.617673:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177242.617674:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41aa00 x1872934856354432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177242.617676:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177242.617680:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177242.617685:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177242.617690:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177242.617692:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000008:00100000:0.0:1786177242.617695:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000001:3.0:1786177242.617698:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000008:00000001:0.0:1786177242.617699:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177242.617703:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177242.617704:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
00000100:00000010:0.0:1786177242.617707:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b100.
02000000:00000001:3.0:1786177242.617710:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000040:0.0:1786177242.617712:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177242.617714:0:2391:0:(genops.c:1238:class_import_put()) Process entered
02000000:00000001:0.0:1786177242.617716:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000020:00000040:3.0:1786177242.617719:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177242.617720:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177242.617725:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00000001:0.0:1786177242.617726:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177242.617729:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:3.0:1786177242.617731:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
02000000:00000001:0.0:1786177242.617732:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177242.617739:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000010:0.0:1786177242.617740:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000100:00000001:0.0:1786177242.617746:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177242.617750:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00100000:0.0:1786177242.617752:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:3.0:1786177242.617755:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41aa00.
00000100:00000001:3.0:1786177242.617762:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177242.617766:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177242.617767:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177242.617770:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.617773:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177242.617777:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000fa632fb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.617785:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177242.617788:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177242.617790:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.617792:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1786177242.617796:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177242.617796:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177242.617799:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1786177242.617804:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000040:3.0:1786177242.617805:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177242.617807:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177242.617811:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177242.617818:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177242.617819:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177242.617822:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177242.617823:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7465
00000400:00000200:0.0:1786177242.617827:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786177242.617832:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000400:00000200:0.0:1786177242.617835:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:3.0:1786177242.617836:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1786177242.617841:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
02000000:00000001:3.0:1786177242.617841:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177242.617846:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354560:192.168.202.110@tcp:38:kworker.0
00000100:00000040:0.0:1786177242.617850:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177242.617850:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177242.617854:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1786177242.617856:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1786177242.617857:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177242.617861:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:0.0:1786177242.617862:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000010:3.0:1786177242.617865:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:0.0:1786177242.617866:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177242.617869:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177242.617872:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177242.617876:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000400:00000200:3.0:1786177242.617876:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177242.617880:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000020:00000001:0.0:1786177242.617881:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177242.617886:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000400:00000010:3.0:1786177242.617886:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000020:00000001:0.0:1786177242.617890:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000200:3.0:1786177242.617891:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856354560, portal 10
00000020:00000001:0.0:1786177242.617895:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:3.0:1786177242.617895:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177242.617897:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761666432 : -114547947885184 : ffff97d1ba41ad80)
00000100:00000040:0.0:1786177242.617903:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:3.0:1786177242.617908:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177242.617909:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177242.617915:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177242.617920:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177242.617921:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:0.0:1786177242.617925:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000040:3.0:1786177242.617926:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177242.617930:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00080000:0.0:1786177242.617934:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000200:3.0:1786177242.617935:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856354560, offset 0
00000100:00000001:0.0:1786177242.617941:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177242.617944:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786177242.617946:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177242.617950:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177242.617950:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786177242.617955:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177242.617955:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1786177242.617961:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:3.0:1786177242.617961:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00100000:0.0:1786177242.617966:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:3.0:1786177242.617966:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177242.617971:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000008:00000001:0.0:1786177242.617973:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177242.617974:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1786177242.617978:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177242.617979:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254941:254941:256:4294967295] 192.168.202.110@tcp LPNI seq info [254930:254930:8:4294967295]
00000100:00000010:0.0:1786177242.617985:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419180.
00000400:00000200:3.0:1786177242.617985:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:0.0:1786177242.617991:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000400:00000200:3.0:1786177242.617995:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
02000000:00000001:0.0:1786177242.617997:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177242.618001:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177242.618002:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177242.618008:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:3.0:1786177242.618008:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
02000000:00000001:0.0:1786177242.618012:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:3.0:1786177242.618015:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:0.0:1786177242.618017:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177242.618020:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
02000000:00000010:0.0:1786177242.618024:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000800:00000200:3.1:1786177242.618025:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177242.618030:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1786177242.618032:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1786177242.618034:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177242.618038:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000100:00100000:0.0:1786177242.618043:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177242.618043:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177242.618051:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177242.618055:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177242.618058:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177242.618061:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177242.618066:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177242.618084:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177242.618085:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177242.618090:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177242.618090:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177242.618101:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005a89d8ab] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618101:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.618103:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177242.618108:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177242.618114:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000100:00000001:0.0:1786177242.618119:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177242.618119:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177242.618124:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302138378
00000100:00000040:0.0:1786177242.618128:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7476)
00000100:00000001:3.0:1786177242.618128:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302142157
00000100:00000001:3.0:1786177242.618133:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302138378
00000100:00000001:3.0:1786177242.618135:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302142157
00000100:00000001:3.0:1786177242.618137:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177242.618139:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.618140:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177242.618144:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177242/real 1786177242]  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177242.618149:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177242.618150:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177242.618152:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177242.618154:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177242.618155:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177242.618157:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177242.618160:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177242.618167:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618173:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177242.618174:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618176:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177242.618177:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177242.618180:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618184:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177242.618187:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618192:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177242.618195:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618197:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177242.618198:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618199:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177242.618201:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177242.618202:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177242.618205:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177242.618206:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177242.618208:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177242.618213:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177242.618216:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177242.618219:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618228:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177242.618229:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177242.618231:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177242.618232:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177242.618233:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177242.618235:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177242.618237:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177242.618238:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177242.618239:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177242.618240:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177242.618242:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177242.618243:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177242.618245:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177242.618247:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177242.618249:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177242.618250:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856354688, portal 10
00000100:00000001:3.0:1786177242.618252:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177242.618253:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761668224 : -114547947883392 : ffff97d1ba41b480)
00000100:00000040:3.0:1786177242.618257:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618262:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177242.618264:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177242.618265:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177242.618267:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856354688, offset 0
00000400:00000200:3.0:1786177242.618269:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177242.618272:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177242.618274:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177242.618275:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177242.618277:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177242.618278:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177242.618280:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177242.618283:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254942:254942:256:4294967295] 192.168.202.110@tcp LPNI seq info [254931:254931:8:4294967295]
00000400:00000200:3.0:1786177242.618286:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177242.618290:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177242.618293:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177242.618296:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177242.618298:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177242.618300:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786177242.618303:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177242.618304:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1786177242.618308:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177242.618309:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177242.618311:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177242.618313:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177242.618315:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177242.618316:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177242.618318:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177242.618320:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618342:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177242.618345:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618350:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.618351:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177242.618354:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177242.618356:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177242.618358:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177242.618360:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302138503
00000100:00000001:3.0:1786177242.618362:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302142157
00000100:00000001:3.0:1786177242.618363:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302138503
00000100:00000001:3.0:1786177242.618365:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302142157
00000100:00000001:3.0:1786177242.618366:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177242.618368:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.618369:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177242.618372:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177242/real 1786177242]  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177242.618377:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177242.618378:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177242.618379:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177242.618381:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177242.618382:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177242.618383:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177242.618385:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177242.618391:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618395:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177242.618396:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618397:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177242.618398:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177242.618400:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618404:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177242.618406:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618410:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177242.618412:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618413:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177242.618414:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618415:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177242.618417:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177242.618419:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177242.618422:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177242.618423:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177242.618426:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177242.618431:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177242.618433:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177242.618435:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618439:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177242.618440:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177242.618441:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177242.618442:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177242.618444:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177242.618445:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177242.618447:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177242.618448:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177242.618449:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177242.618450:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177242.618454:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177242.618456:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177242.618458:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177242.618460:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177242.618462:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177242.618464:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856354816, portal 4
00000100:00000001:3.0:1786177242.618466:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177242.618467:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761667328 : -114547947884288 : ffff97d1ba41b100)
00000100:00000040:3.0:1786177242.618471:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618475:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177242.618477:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177242.618479:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177242.618480:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856354816, offset 0
00000400:00000200:3.0:1786177242.618483:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177242.618486:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177242.618488:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177242.618490:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177242.618492:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177242.618494:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177242.618495:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177242.618498:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254943:254943:256:4294967295] 192.168.202.110@tcp LPNI seq info [254932:254932:8:4294967295]
00000400:00000200:3.0:1786177242.618501:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177242.618505:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177242.618508:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177242.618527:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177242.618531:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177242.618533:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786177242.618536:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177242.618538:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1786177242.618540:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177242.618542:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177242.618544:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177242.618547:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177242.618549:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177242.618550:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177242.618552:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177242.618554:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618562:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177242.618565:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618573:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.618574:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177242.618578:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177242.618582:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177242.618586:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177242.618588:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302138379
00000100:00000001:3.0:1786177242.618591:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302142157
00000100:00000001:3.0:1786177242.618593:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302138379
00000100:00000001:3.0:1786177242.618596:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302142157
00000100:00000001:3.0:1786177242.618598:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177242.618600:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.618602:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177242.618606:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177242/real 1786177242]  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177242.618614:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177242.618615:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177242.618616:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177242.618618:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177242.618620:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177242.618621:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177242.618624:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177242.618629:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618634:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177242.618635:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618636:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177242.618637:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177242.618639:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618643:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177242.618646:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618651:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177242.618653:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618654:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177242.618655:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618656:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177242.618657:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177242.618659:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177242.618661:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177242.618662:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177242.618664:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177242.618669:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177242.618671:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618672:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177242.618674:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41ad80 x1872934856354560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618679:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177242.618680:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177242.618682:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177242.618684:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177242.618685:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177242.618686:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177242.618687:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177242.618689:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:3.0:1786177242.618690:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177242.618692:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177242.618693:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41ad80.
00000100:00000001:3.0:1786177242.618695:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177242.618696:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618698:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177242.618700:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b480 x1872934856354688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618705:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177242.618706:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177242.618707:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177242.618708:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177242.618709:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177242.618709:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177242.618710:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177242.618711:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:3.0:1786177242.618712:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177242.618713:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177242.618714:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b480.
00000100:00000001:3.0:1786177242.618715:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177242.618716:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618717:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177242.618719:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b100 x1872934856354816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618722:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177242.618723:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177242.618724:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177242.618725:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177242.618726:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177242.618726:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177242.618727:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177242.618728:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:3.0:1786177242.618729:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177242.618730:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177242.618731:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b100.
00000100:00000001:3.0:1786177242.618732:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177242.618733:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618734:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618736:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177242.618737:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.618739:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177242.618740:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177242.618741:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177242.618743:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618749:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177242.618750:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177242.618751:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177242.618752:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177242.618753:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177242.618755:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177242.618757:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177242.618759:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177242.618760:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177242.618761:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177242.618763:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177242.618765:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177242.618767:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177242.618769:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177242.618771:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177242.618773:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856354944, portal 4
00000100:00000001:3.0:1786177242.618775:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177242.618776:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761659264 : -114547947892352 : ffff97d1ba419180)
00000100:00000040:3.0:1786177242.618780:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618784:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177242.618786:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177242.618787:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177242.618789:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856354944, offset 0
00000400:00000200:3.0:1786177242.618791:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177242.618794:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177242.618796:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177242.618798:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177242.618801:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177242.618803:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177242.618805:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177242.618807:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254944:254944:256:4294967295] 192.168.202.110@tcp LPNI seq info [254933:254933:8:4294967295]
00000400:00000200:3.0:1786177242.618810:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177242.618814:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177242.618817:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177242.618819:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177242.618822:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177242.618823:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1786177242.618826:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177242.618828:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1786177242.618831:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177242.618832:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177242.618834:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177242.618837:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177242.618838:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177242.618840:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177242.618841:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177242.618844:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618849:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177242.618851:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618856:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.618857:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177242.618859:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177242.618861:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177242.618864:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177242.618865:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302138387
00000100:00000001:3.0:1786177242.618867:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302142157
00000100:00000001:3.0:1786177242.618869:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302138387
00000100:00000001:3.0:1786177242.618871:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302142157
00000100:00000001:3.0:1786177242.618873:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177242.618874:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.618876:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177242.618878:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177242/real 1786177242]  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177242.618883:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177242.618883:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177242.618884:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177242.618886:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177242.618887:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177242.618888:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177242.618890:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177242.618895:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618900:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177242.618900:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618901:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177242.618902:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177242.618904:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618908:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177242.618910:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618914:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177242.618916:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618917:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177242.618917:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618919:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177242.618920:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177242.618922:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177242.618924:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177242.618926:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177242.618928:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177242.618933:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856354944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177242.618935:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618936:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177242.618938:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419180 x1872934856354944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177242.618941:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177242.618943:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177242.618944:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177242.618945:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177242.618946:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177242.618947:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177242.618948:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177242.618950:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:3.0:1786177242.618952:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177242.618953:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177242.618955:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419180.
00000100:00000001:3.0:1786177242.618957:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177242.618959:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618961:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177242.618964:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177242.618965:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177242.618967:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177242.618969:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177243.192590:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177243.192596:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177243.192601:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177243.192602:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177243.192605:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177243.192606:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177243.192674:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177243.192678:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177243.192680:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177243.640613:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177243.640619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177243.640621:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177243.640625:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177243.640628:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177243.640631:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177243.640633:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177243.640636:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177243.640638:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177244.216588:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177244.216593:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177244.216598:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177244.216599:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177244.216601:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177244.216604:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177244.216605:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177244.216608:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177244.216610:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177244.216612:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177244.216614:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177244.216615:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177244.216668:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177244.216671:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177244.216674:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177244.664592:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177244.664600:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177244.664602:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177244.664608:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177244.664611:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177244.664615:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177244.664618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177244.664621:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177244.664624:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177245.240634:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177245.240640:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177245.240644:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177245.240645:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177245.240647:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177245.240648:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177245.240708:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177245.240712:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177245.240715:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177245.688628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177245.688634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177245.688636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177245.688640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177245.688643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177245.688646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177245.688649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177245.688651:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177245.688652:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177246.264594:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177246.264600:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177246.264604:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177246.264606:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177246.264608:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177246.264609:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177246.264672:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177246.264677:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177246.264681:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177246.456621:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302142157, at 4302142208
00000100:00000040:1.0:1786177246.456621:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302142157, at 4302142208
00000100:00000010:1.0:1786177246.456637:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37600.
00000100:00000010:0.0:1786177246.456642:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37300.
00000100:00000001:2.0:1786177246.712610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177246.712617:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177246.712619:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177246.712623:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177246.712625:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177246.712629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177246.712631:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177246.712633:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177246.712635:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177247.288631:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177247.288637:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177247.288641:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177247.288642:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177247.288644:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177247.288645:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177247.288693:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177247.288696:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177247.288699:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177247.736619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177247.736625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786177247.736627:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786177247.736627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177247.736634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:0.0:1786177247.736637:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177247.736638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177247.736641:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1786177247.736642:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177247.736646:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1786177247.736647:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177247.736648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177247.736653:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177247.736655:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177247.736664:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7471
00000400:00000200:0.0:1786177247.736673:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177247.736681:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177247.736685:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177247.736689:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.736693:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177247.736696:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.736699:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177247.736703:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177247.736705:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177247.736708:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177247.736712:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177247.736716:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177247.736718:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177247.736721:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.736724:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177247.736726:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.736729:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177247.736732:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177247.736735:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177247.736738:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177247.736741:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177247.736744:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177247.736747:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177247.736748:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177247.736751:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177247.736759:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418e00.
00000020:00000040:0.0:1786177247.736763:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177247.736767:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177247.736769:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177247.736772:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177247.736775:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177247.736779:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177247.736788:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:0.0:1786177247.736795:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177247.736804:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177247.736815:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177247.736820:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177247.736825:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003428892b] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177247.736849:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177247.736853:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177247.736857:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177247.736858:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177247.736860:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177247.736863:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177247.736866:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7471
00000400:00000200:0.0:1786177247.736870:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177247.736874:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177247.736877:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177247.736881:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.736884:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177247.736887:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.736890:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177247.736892:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177247.736894:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:3.0:1786177247.736894:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1786177247.736899:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177247.736900:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1786177247.736902:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:3.0:1786177247.736906:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1786177247.736907:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177247.736911:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177247.736918:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:3.0:1786177247.736918:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177247.736923:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177247.736929:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.736935:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177247.736941:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:3.0:1786177247.736942:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177247.736946:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177247.736949:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1786177247.736951:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1786177247.736956:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177247.736957:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177247.736961:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177247.736963:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177247.736969:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1786177247.736970:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856355072:192.168.202.110@tcp:250:kworker.0
02000000:00000001:0.0:1786177247.736976:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.736979:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1786177247.736980:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:3.0:1786177247.736984:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:0.0:1786177247.736986:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1786177247.736989:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1786177247.736991:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177247.736993:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1786177247.736995:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177247.737000:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000010:0.0:1786177247.737001:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418a80.
02000000:00000001:3.0:1786177247.737005:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177247.737007:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177247.737011:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177247.737013:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177247.737015:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177247.737017:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177247.737020:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1786177247.737021:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
02000000:00000001:0.0:1786177247.737022:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177247.737030:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000400:00000010:3.0:1786177247.737031:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177247.737035:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177247.737041:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856355072, portal 25
00000100:00100000:0.0:1786177247.737045:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177247.737046:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177247.737049:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761658368 : -114547947893248 : ffff97d1ba418e00)
00000100:00000040:3.0:1786177247.737061:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177247.737062:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177247.737067:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.737077:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177247.737078:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000606c34cb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177247.737084:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177247.737090:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177247.737096:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856355072, offset 0
00000100:00000001:0.0:1786177247.737098:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177247.737103:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1786177247.737109:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177247.737110:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177247.737114:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177247.737116:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177247.737120:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177247.737121:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177247.737126:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7471
00000400:00000200:3.0:1786177247.737132:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177247.737133:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177247.737140:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1786177247.737143:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:0.0:1786177247.737144:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1786177247.737151:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000040:0.0:1786177247.737152:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.737157:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177247.737160:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1786177247.737163:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177247.737167:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.737169:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177247.737174:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:3.0:1786177247.737176:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254945:254945:256:4294967295] 192.168.202.110@tcp LPNI seq info [254934:254934:8:4294967295]
00000020:00000040:0.0:1786177247.737177:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177247.737181:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177247.737185:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000400:00000200:3.0:1786177247.737186:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:0.0:1786177247.737190:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177247.737194:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177247.737200:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.737205:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177247.737206:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1786177247.737210:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.737214:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.0:1786177247.737219:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:0.0:1786177247.737220:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786177247.737227:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000010:3.0:1786177247.737228:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000100:00000001:0.0:1786177247.737231:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177247.737234:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177247.737238:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:3.0:1786177247.737239:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:0.0:1786177247.737243:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177247.737246:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:0.0:1786177247.737248:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177247.737253:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000800:00000200:3.1:1786177247.737255:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000008:00000001:0.0:1786177247.737260:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177247.737261:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:0.0:1786177247.737265:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1786177247.737270:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000010:0.0:1786177247.737293:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419180.
00000020:00000040:0.0:1786177247.737297:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177247.737300:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177247.737301:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177247.737303:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177247.737305:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177247.737308:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1786177247.737308:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
02000000:00000010:0.0:1786177247.737316:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000400:00000200:3.0:1786177247.737318:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1786177247.737321:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177247.737327:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00100000:0.0:1786177247.737333:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177247.737336:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177247.737341:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177247.737348:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177247.737356:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177247.737357:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177247.737361:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177247.737370:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005a89d8ab] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.737376:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177247.737383:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177247.737384:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177247.737389:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177247.737398:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177247.737402:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177247.737403:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177247.737404:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177247.737406:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177247.737411:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177247.737414:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7471
00000100:00000001:3.0:1786177247.737427:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177247.737429:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177247.737436:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177247.737440:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1786177247.737440:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177247.737445:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302147277
00000100:00000040:0.0:1786177247.737447:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.737452:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:3.0:1786177247.737453:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37de0.
00000100:00000040:0.0:1786177247.737455:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.737460:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177247.737463:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000001:0.0:1786177247.737464:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177247.737467:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177247.737470:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177247.737473:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:3.0:1786177247.737473:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302147277
00000020:00000001:0.0:1786177247.737479:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000010:3.0:1786177247.737481:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37c00.
00000020:00000001:0.0:1786177247.737483:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:3.0:1786177247.737487:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:0.0:1786177247.737488:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.737493:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177247.737493:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1786177247.737497:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177247.737499:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177247.737501:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177247.737503:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000020:00000040:0.0:1786177247.737508:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00100000:3.0:1786177247.737541:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177247/real 1786177247]  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177247.737548:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177247.737555:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177247.737560:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000001:3.0:1786177247.737563:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:0.0:1786177247.737565:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000001:3.0:1786177247.737567:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:0.0:1786177247.737570:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177247.737572:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00000001:0.0:1786177247.737574:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:3.0:1786177247.737576:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000008:00100000:0.0:1786177247.737578:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000010:3.0:1786177247.737579:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177247.737584:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000008:00000001:0.0:1786177247.737585:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177247.737590:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177247.737591:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1786177247.737595:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b100.
00000020:00000040:0.0:1786177247.737601:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177247.737607:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177247.737610:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177247.737611:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177247.737615:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177247.737619:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177247.737623:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.737624:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177247.737628:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:0.0:1786177247.737630:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000001:3.0:1786177247.737631:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177247.737636:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177247.737638:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177247.737644:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177247.737648:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177247.737667:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177247.737672:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177247.737676:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177247.737678:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177247.737685:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000fa632fb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.737696:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177247.737704:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177247.737706:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.737708:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:0.0:1786177247.737710:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177247.737712:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177247.737717:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177247.737718:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177247.737719:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177247.737723:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177247.737724:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177247.737728:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177247.737731:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177247.737733:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7471
00000100:00080000:3.0:1786177247.737740:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000400:00000200:0.0:1786177247.737741:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177247.737746:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000400:00000200:0.0:1786177247.737750:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177247.737755:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177247.737756:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177247.737761:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177247.737768:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177247.737773:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00100000:3.0:1786177247.737778:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856355072:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177247.737779:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177247.737784:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177247.737787:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:3.0:1786177247.737787:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1786177247.737791:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177247.737793:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1786177247.737795:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177247.737800:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177247.737800:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418e00 x1872934856355072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177247.737807:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786177247.737817:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177247.737819:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000001:0.0:1786177247.737822:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1786177247.737824:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000040:0.0:1786177247.737827:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000010:3.0:1786177247.737830:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
00000100:00000001:0.0:1786177247.737831:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177247.737835:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000040:0.0:1786177247.737836:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000020:00000001:3.0:1786177247.737839:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000100:00080000:0.0:1786177247.737843:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000020:00000040:3.0:1786177247.737843:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177247.737848:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00000001:0.0:1786177247.737850:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177247.737852:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:0.0:1786177247.737853:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177247.737859:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177247.737860:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177247.737863:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
02000000:00000001:0.0:1786177247.737864:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177247.737867:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418e00.
00000100:00000001:0.0:1786177247.737868:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177247.737872:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177247.737873:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000008:00000001:0.0:1786177247.737876:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.737877:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177247.737880:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.737882:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:0.0:1786177247.737886:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b480.
00000020:00000040:0.0:1786177247.737890:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177247.737891:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:0.0:1786177247.737894:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177247.737895:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177247.737899:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.737900:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177247.737903:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177247.737904:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
02000000:00000001:0.0:1786177247.737907:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177247.737908:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:0.0:1786177247.737911:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177247.737918:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6000.
00000100:00000040:3.0:1786177247.737918:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177247.737924:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.737933:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00100000:0.0:1786177247.737935:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177247.737937:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177247.737943:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177247.737946:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177247.737950:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1786177247.737951:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177247.737955:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177247.737958:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856355200:192.168.202.110@tcp:38:kworker.0
00000100:00000040:0.0:1786177247.737964:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001215e9e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.737964:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177247.737968:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177247.737970:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177247.737974:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177247.737979:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177247.737983:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177247.737984:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177247.737990:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177247.737992:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7481)
00000400:00000010:3.0:1786177247.737994:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177247.738000:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177247.738003:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856355200, portal 10
00000100:00000001:3.0:1786177247.738006:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177247.738008:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761657472 : -114547947894144 : ffff97d1ba418a80)
00000100:00000040:3.0:1786177247.738016:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738024:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177247.738028:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177247.738031:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177247.738034:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856355200, offset 0
00000400:00000200:3.0:1786177247.738039:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177247.738044:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177247.738050:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177247.738054:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177247.738057:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177247.738060:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177247.738063:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177247.738068:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254946:254946:256:4294967295] 192.168.202.110@tcp LPNI seq info [254935:254935:8:4294967295]
00000400:00000200:3.0:1786177247.738073:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177247.738081:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177247.738086:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177247.738089:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177247.738094:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177247.738097:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786177247.738101:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177247.738104:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786177247.738108:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177247.738110:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177247.738114:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177247.738118:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177247.738121:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177247.738123:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177247.738126:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177247.738130:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738138:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177247.738141:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738148:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.738149:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177247.738153:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177247.738157:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177247.738161:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177247.738163:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302143498
00000100:00000001:3.0:1786177247.738166:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302147277
00000100:00000001:3.0:1786177247.738168:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302143498
00000100:00000001:3.0:1786177247.738170:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302147277
00000100:00000001:3.0:1786177247.738173:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177247.738175:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.738177:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177247.738181:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177247/real 1786177247]  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177247.738189:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177247.738190:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177247.738192:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177247.738195:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177247.738196:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177247.738198:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177247.738201:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177247.738210:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738217:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177247.738219:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.738221:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177247.738222:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177247.738226:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738234:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177247.738238:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738245:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177247.738247:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.738249:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177247.738250:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.738252:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177247.738254:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177247.738257:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177247.738260:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177247.738262:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177247.738267:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177247.738290:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856355200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177247.738294:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177247.738298:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738305:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177247.738306:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177247.738309:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177247.738310:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177247.738313:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177247.738315:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856355328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177247.738319:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177247.738321:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177247.738322:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177247.738324:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177247.738328:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177247.738330:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177247.738334:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177247.738336:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177247.738340:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177247.738343:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856355328, portal 10
00000100:00000001:3.0:1786177247.738345:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177247.738346:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761659264 : -114547947892352 : ffff97d1ba419180)
00000100:00000040:3.0:1786177247.738353:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738360:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177247.738362:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177247.738364:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177247.738367:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856355328, offset 0
00000400:00000200:3.0:1786177247.738372:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177247.738375:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177247.738379:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177247.738382:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177247.738385:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177247.738388:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177247.738390:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177247.738393:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254947:254947:256:4294967295] 192.168.202.110@tcp LPNI seq info [254936:254936:8:4294967295]
00000400:00000200:3.0:1786177247.738398:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177247.738404:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177247.738409:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177247.738412:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177247.738417:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177247.738419:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786177247.738424:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177247.738427:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786177247.738431:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177247.738433:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177247.738436:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177247.738440:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177247.738443:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177247.738445:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177247.738447:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177247.738451:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738458:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177247.738462:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738469:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.738471:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177247.738473:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177247.738476:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177247.738479:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177247.738482:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302143623
00000100:00000001:3.0:1786177247.738484:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302147277
00000100:00000001:3.0:1786177247.738487:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302143623
00000100:00000001:3.0:1786177247.738489:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302147277
00000100:00000001:3.0:1786177247.738491:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177247.738493:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.738494:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177247.738498:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177247/real 1786177247]  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177247.738506:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177247.738507:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177247.738508:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177247.738536:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177247.738538:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177247.738540:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177247.738544:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177247.738554:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738561:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177247.738562:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.738564:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177247.738566:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177247.738569:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738577:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177247.738581:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738587:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177247.738590:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.738592:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177247.738594:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.738595:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177247.738597:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177247.738600:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177247.738603:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177247.738606:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177247.738610:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177247.738619:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856355328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177247.738623:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177247.738626:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738633:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177247.738635:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177247.738637:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177247.738639:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177247.738641:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177247.738644:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856355456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177247.738648:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177247.738650:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177247.738651:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177247.738653:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177247.738657:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177247.738659:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177247.738663:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177247.738666:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177247.738669:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177247.738672:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856355456, portal 4
00000100:00000001:3.0:1786177247.738675:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177247.738676:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761667328 : -114547947884288 : ffff97d1ba41b100)
00000100:00000040:3.0:1786177247.738682:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738689:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177247.738691:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177247.738694:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177247.738696:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856355456, offset 0
00000400:00000200:3.0:1786177247.738700:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177247.738704:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177247.738707:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177247.738710:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177247.738713:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177247.738715:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177247.738718:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177247.738721:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254948:254948:256:4294967295] 192.168.202.110@tcp LPNI seq info [254937:254937:8:4294967295]
00000400:00000200:3.0:1786177247.738726:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177247.738732:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177247.738737:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177247.738740:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177247.738744:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177247.738747:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786177247.738751:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177247.738753:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786177247.738757:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177247.738760:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177247.738763:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177247.738766:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177247.738769:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177247.738771:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177247.738773:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177247.738777:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738791:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177247.738794:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738802:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.738803:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177247.738806:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177247.738809:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177247.738812:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177247.738814:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302143499
00000100:00000001:3.0:1786177247.738816:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302147277
00000100:00000001:3.0:1786177247.738819:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302143499
00000100:00000001:3.0:1786177247.738821:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302147277
00000100:00000001:3.0:1786177247.738823:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177247.738825:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.738827:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177247.738831:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177247/real 1786177247]  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177247.738839:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177247.738841:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177247.738843:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177247.738845:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177247.738847:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177247.738850:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177247.738853:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177247.738862:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738870:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177247.738871:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.738873:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177247.738875:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177247.738878:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738886:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177247.738890:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738897:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177247.738900:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.738901:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177247.738903:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.738904:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177247.738906:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177247.738909:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177247.738912:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177247.738915:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177247.738919:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177247.738927:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856355456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177247.738931:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.738933:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177247.738936:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418a80 x1872934856355200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738944:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177247.738945:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177247.738948:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177247.738950:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177247.738952:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177247.738954:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177247.738956:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177247.738958:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:3.0:1786177247.738960:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177247.738962:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177247.738964:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418a80.
00000100:00000001:3.0:1786177247.738967:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177247.738968:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.738970:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177247.738974:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419180 x1872934856355328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.738981:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177247.738982:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177247.738984:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177247.738986:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177247.738987:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177247.738988:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177247.738990:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177247.738991:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:3.0:1786177247.738993:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177247.738994:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177247.738995:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419180.
00000100:00000001:3.0:1786177247.738997:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177247.738998:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.739000:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177247.739003:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b100 x1872934856355456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.739009:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177247.739010:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177247.739011:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177247.739013:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177247.739015:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177247.739016:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177247.739018:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177247.739019:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:3.0:1786177247.739021:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177247.739023:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177247.739024:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b100.
00000100:00000001:3.0:1786177247.739026:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177247.739027:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.739029:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.739033:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177247.739035:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.739037:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177247.739038:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177247.739040:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177247.739044:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.739051:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177247.739052:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177247.739054:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177247.739056:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177247.739058:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177247.739061:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856355584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177247.739064:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177247.739067:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177247.739068:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177247.739070:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177247.739073:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177247.739075:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177247.739079:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177247.739082:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177247.739085:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177247.739088:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856355584, portal 4
00000100:00000001:3.0:1786177247.739090:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177247.739092:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761668224 : -114547947883392 : ffff97d1ba41b480)
00000100:00000040:3.0:1786177247.739098:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.739105:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177247.739107:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177247.739110:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177247.739112:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856355584, offset 0
00000400:00000200:3.0:1786177247.739116:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177247.739120:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177247.739124:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177247.739127:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177247.739129:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177247.739132:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177247.739134:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177247.739138:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254949:254949:256:4294967295] 192.168.202.110@tcp LPNI seq info [254938:254938:8:4294967295]
00000400:00000200:3.0:1786177247.739142:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177247.739148:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177247.739153:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177247.739156:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177247.739160:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177247.739163:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1786177247.739167:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177247.739170:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1786177247.739174:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177247.739176:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177247.739179:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177247.739182:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177247.739185:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177247.739187:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177247.739190:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177247.739193:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.739202:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177247.739205:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.739212:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.739214:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177247.739217:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177247.739220:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177247.739224:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177247.739226:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302143507
00000100:00000001:3.0:1786177247.739228:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302147277
00000100:00000001:3.0:1786177247.739230:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302143507
00000100:00000001:3.0:1786177247.739232:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302147277
00000100:00000001:3.0:1786177247.739235:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177247.739236:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.739238:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177247.739240:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177247/real 1786177247]  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177247.739246:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177247.739247:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177247.739248:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177247.739250:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177247.739251:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177247.739252:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177247.739255:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177247.739260:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.739265:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177247.739265:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.739267:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177247.739267:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177247.739270:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.739291:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177247.739295:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.739300:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177247.739302:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.739303:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177247.739304:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.739305:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177247.739306:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177247.739308:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177247.739310:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177247.739311:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177247.739313:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177247.739318:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856355584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177247.739321:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.739322:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177247.739324:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b480 x1872934856355584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177247.739328:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177247.739329:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177247.739330:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177247.739331:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177247.739332:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177247.739333:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177247.739334:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177247.739336:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6000.
02000000:00000001:3.0:1786177247.739337:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177247.739338:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177247.739339:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b480.
00000100:00000001:3.0:1786177247.739341:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177247.739342:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.739343:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177247.739345:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177247.739346:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177247.739348:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177247.739349:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177248.312632:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177248.312638:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177248.312642:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177248.312643:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177248.312646:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177248.312647:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177248.312730:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177248.312733:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177248.312736:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177248.760633:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177248.760639:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177248.760641:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177248.760645:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177248.760648:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177248.760651:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177248.760653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177248.760655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177248.760657:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177249.336631:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177249.336637:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177249.336642:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177249.336643:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177249.336645:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177249.336646:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177249.336709:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177249.336713:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177249.336716:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177249.784617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177249.784624:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177249.784626:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177249.784632:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177249.784636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177249.784640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177249.784644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177249.784647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177249.784650:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177250.360612:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177250.360617:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177250.360621:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177250.360622:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177250.360625:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177250.360626:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177250.360677:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177250.360680:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177250.360683:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177250.808642:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177250.808651:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177250.808653:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177250.808659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177250.808663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177250.808667:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177250.808670:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177250.808673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177250.808676:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177251.384660:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177251.384668:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177251.384670:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177251.384673:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177251.384677:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177251.384679:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177251.384681:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177251.384682:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177251.384686:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177251.384688:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177251.384689:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177251.384690:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177251.384691:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177251.384692:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177251.384693:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177251.384695:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177251.384697:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177251.384697:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177251.384751:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177251.384754:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177251.384758:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177251.576623:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302147277, at 4302147328
00000100:00000040:1.0:1786177251.576623:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302147277, at 4302147328
00000100:00000010:1.0:1786177251.576642:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37c00.
00000100:00000010:0.0:1786177251.576644:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37de0.
00000100:00000001:2.0:1786177251.832616:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177251.832623:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177251.832625:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177251.832630:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177251.832633:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177251.832636:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177251.832638:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177251.832640:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177251.832642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177252.408616:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177252.408624:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177252.408629:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177252.408631:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177252.408634:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177252.408635:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177252.408703:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177252.408705:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177252.408708:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786177252.856645:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177252.856653:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177252.856655:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1786177252.856658:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1786177252.856659:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177252.856664:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177252.856664:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786177252.856669:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177252.856674:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7476
00000400:00000200:3.0:1786177252.856682:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177252.856687:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00100000:2.0:1786177252.856688:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:3.0:1786177252.856689:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177252.856695:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177252.856696:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177252.856698:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177252.856701:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177252.856702:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177252.856704:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:2.0:1786177252.856708:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177252.856712:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177252.856716:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177252.856716:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177252.856721:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177252.856726:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177252.856729:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177252.856732:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177252.856735:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177252.856739:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177252.856742:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177252.856744:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177252.856746:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177252.856749:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177252.856751:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177252.856753:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177252.856756:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177252.856759:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177252.856761:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177252.856763:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177252.856764:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177252.856772:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d500.
00000020:00000040:3.0:1786177252.856776:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177252.856779:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177252.856780:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177252.856783:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177252.856786:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177252.856790:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177252.856798:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:3.0:1786177252.856805:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177252.856815:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177252.856826:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177252.856831:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177252.856836:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7c36393] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177252.856861:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177252.856874:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177252.856878:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177252.856881:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177252.856882:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177252.856886:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177252.856889:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7476
00000400:00000200:3.0:1786177252.856893:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177252.856894:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:3.0:1786177252.856896:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1786177252.856900:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1786177252.856900:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177252.856905:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177252.856906:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177252.856911:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177252.856917:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000040:0.0:1786177252.856918:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177252.856920:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177252.856927:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177252.856930:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177252.856934:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:0.0:1786177252.856938:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177252.856942:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177252.856946:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177252.856949:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1786177252.856949:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786177252.856956:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
02000000:00000001:0.0:1786177252.856957:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177252.856963:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00100000:0.0:1786177252.856965:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856355712:192.168.202.110@tcp:250:kworker.0
00000100:00000040:3.0:1786177252.856971:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177252.856974:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1786177252.856975:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177252.856978:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:0.0:1786177252.856980:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1786177252.856981:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177252.856986:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
02000000:00000001:0.0:1786177252.856987:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177252.856988:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1786177252.856993:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1786177252.856993:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177252.856998:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:0.0:1786177252.857002:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5800.
02000000:00000001:3.0:1786177252.857003:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177252.857009:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177252.857009:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177252.857015:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177252.857019:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1786177252.857023:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:3.0:1786177252.857023:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177252.857027:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177252.857031:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000100:00000010:3.0:1786177252.857032:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ea00.
00000020:00000040:3.0:1786177252.857037:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177252.857038:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177252.857039:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1786177252.857041:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000001:3.0:1786177252.857048:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:0.0:1786177252.857050:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856355712, portal 25
02000000:00000001:3.0:1786177252.857051:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177252.857054:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.857056:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177252.857061:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
02000000:00000010:3.0:1786177252.857061:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786177252.857067:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177252.857074:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177252.857075:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177252.857109:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177252.857110:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1786177252.857111:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177252.857118:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:3.0:1786177252.857118:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053162887] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1786177252.857125:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000001:3.0:1786177252.857128:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786177252.857131:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856355712, offset 0
00000100:00080000:3.0:1786177252.857132:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177252.857138:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177252.857140:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177252.857142:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1786177252.857145:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177252.857147:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177252.857153:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7476
00000400:00000200:0.0:1786177252.857157:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177252.857162:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177252.857168:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177252.857169:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177252.857175:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786177252.857177:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1786177252.857181:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177252.857187:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:3.0:1786177252.857187:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177252.857193:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177252.857196:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1786177252.857198:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177252.857201:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:0.0:1786177252.857203:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000040:3.0:1786177252.857204:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177252.857208:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177252.857211:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786177252.857215:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:0.0:1786177252.857216:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254950:254950:256:4294967295] 192.168.202.110@tcp LPNI seq info [254939:254939:8:4294967295]
00000400:00000200:0.0:1786177252.857222:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1786177252.857238:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177252.857245:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177252.857247:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177252.857250:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177252.857252:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177252.857258:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:0.0:1786177252.857261:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1786177252.857263:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177252.857269:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177252.857272:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177252.857274:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:0.0:1786177252.857279:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:3.0:1786177252.857279:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177252.857285:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177252.857287:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177252.857289:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1786177252.857290:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000100:00000001:3.0:1786177252.857291:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177252.857298:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c000.
00000020:00000040:3.0:1786177252.857302:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000800:00000200:0.0:1786177252.857303:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:3.0:1786177252.857304:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177252.857307:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177252.857310:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:0.0:1786177252.857311:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
02000000:00000001:3.0:1786177252.857311:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177252.857314:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177252.857320:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000800:00000200:0.1:1786177252.857325:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177252.857326:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1786177252.857331:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00100000:3.0:1786177252.857334:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:0.0:1786177252.857344:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177252.857346:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000010:0.0:1786177252.857350:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000100:00000001:3.0:1786177252.857350:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177252.857356:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006fe8fdb7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1786177252.857359:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1786177252.857367:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177252.857370:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1786177252.857371:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:3.0:1786177252.857373:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177252.857377:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177252.857379:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1786177252.857380:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000100:00000001:3.0:1786177252.857381:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:0.0:1786177252.857387:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00080000:3.0:1786177252.857387:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7476
00000400:00000200:3.0:1786177252.857395:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177252.857396:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:3.0:1786177252.857400:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177252.857405:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:0.0:1786177252.857406:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177252.857413:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177252.857420:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177252.857425:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177252.857427:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786177252.857430:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177252.857435:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000040:0.0:1786177252.857436:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1786177252.857437:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177252.857441:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177252.857445:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177252.857450:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177252.857452:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177252.857455:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177252.857458:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177252.857459:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177252.857460:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177252.857465:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177252.857465:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177252.857470:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177252.857472:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177252.857474:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.857476:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177252.857476:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177252.857480:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177252.857483:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177252.857484:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177252.857486:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177252.857490:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:0.0:1786177252.857491:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302152397
00000008:00000001:3.0:1786177252.857492:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177252.857496:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177252.857501:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000010:3.0:1786177252.857502:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c700.
00000020:00000040:3.0:1786177252.857507:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177252.857539:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177252.857541:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177252.857543:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786177252.857544:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
02000000:00000001:3.0:1786177252.857545:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177252.857548:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177252.857553:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:0.0:1786177252.857554:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302152398
00000100:00000001:3.0:1786177252.857556:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177252.857563:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:0.0:1786177252.857564:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134060.
00000100:00000001:0.0:1786177252.857571:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177252.857572:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177252.857576:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.857579:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1786177252.857580:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e56458ae] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.857586:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.857592:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1786177252.857593:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177252.857597:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177252.857599:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177252.857601:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:0.0:1786177252.857603:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177252/real 1786177252]  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1786177252.857603:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177252.857609:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177252.857613:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7476
00000400:00000200:3.0:1786177252.857623:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000001:0.0:1786177252.857625:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177252.857629:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000200:3.0:1786177252.857630:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000010:0.0:1786177252.857634:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000100:00080000:3.0:1786177252.857635:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1786177252.857640:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000100:00000040:3.0:1786177252.857642:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:0.0:1786177252.857645:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:3.0:1786177252.857647:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177252.857650:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:3.0:1786177252.857652:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177252.857657:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:0.0:1786177252.857659:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177252.857662:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177252.857665:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177252.857669:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177252.857672:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177252.857675:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000200:0.0:1786177252.857678:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177252.857679:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177252.857687:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177252.857691:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177252.857696:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177252.857699:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177252.857701:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177252.857703:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:3.0:1786177252.857706:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:0.0:1786177252.857707:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177252.857712:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786177252.857714:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00100000:0.0:1786177252.857720:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177252.857721:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177252.857727:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177252.857731:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177252.857735:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177252.857736:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177252.857742:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000040:0.0:1786177252.857743:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1786177252.857744:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177252.857750:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177252.857754:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177252.857760:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e300.
00000020:00000040:3.0:1786177252.857765:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:0.0:1786177252.857766:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:3.0:1786177252.857769:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1786177252.857772:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177252.857772:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177252.857778:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:0.0:1786177252.857780:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1786177252.857780:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:0.0:1786177252.857784:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1786177252.857785:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.857789:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000010:3.0:1786177252.857792:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:0.0:1786177252.857793:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177252.857800:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:3.0:1786177252.857800:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177252.857808:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00100000:3.0:1786177252.857812:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177252.857814:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177252.857824:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177252.857831:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177252.857836:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177252.857842:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856355712:192.168.202.110@tcp:250:kworker.0
00000100:00000040:3.0:1786177252.857846:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e3752a60] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.857852:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.857858:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786177252.857864:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177252.857866:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856355712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177252.857872:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7486)
00000100:00000001:0.0:1786177252.857884:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177252.857887:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177252.857890:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177252.857894:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177252.857896:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177252.857898:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177252.857900:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177252.857902:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:0.0:1786177252.857906:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177252.857908:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177252.857909:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:0.0:1786177252.857913:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177252.857914:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.857916:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.857923:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177252.857925:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.857927:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177252.857929:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177252.857930:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177252.857934:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.857941:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177252.857943:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177252.857945:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177252.857947:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177252.857949:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177252.857952:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856355840:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177252.857956:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177252.857958:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177252.857960:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177252.857962:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177252.857965:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177252.857967:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177252.857971:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177252.857975:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177252.857978:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177252.857982:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856355840, portal 10
00000100:00000001:0.0:1786177252.857984:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177252.857985:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:0.0:1786177252.857993:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858000:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177252.858002:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177252.858005:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177252.858008:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856355840, offset 0
00000400:00000200:0.0:1786177252.858012:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177252.858016:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177252.858020:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177252.858023:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177252.858026:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177252.858029:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177252.858031:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177252.858034:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254951:254951:256:4294967295] 192.168.202.110@tcp LPNI seq info [254940:254940:8:4294967295]
00000400:00000200:0.0:1786177252.858039:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177252.858046:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177252.858051:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177252.858055:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177252.858059:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177252.858063:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786177252.858067:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177252.858071:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786177252.858075:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177252.858077:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177252.858080:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177252.858083:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177252.858087:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177252.858089:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177252.858093:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177252.858096:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858105:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177252.858108:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858114:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.858116:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177252.858119:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177252.858123:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177252.858127:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177252.858129:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302148618
00000100:00000001:0.0:1786177252.858132:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302152397
00000100:00000001:0.0:1786177252.858134:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302148618
00000100:00000001:0.0:1786177252.858136:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302152398
00000100:00000001:0.0:1786177252.858138:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177252.858140:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.858142:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177252.858146:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177252/real 1786177252]  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177252.858154:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177252.858156:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177252.858158:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177252.858160:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177252.858162:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177252.858165:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177252.858168:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177252.858176:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858190:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177252.858192:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.858193:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177252.858195:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177252.858199:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858207:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177252.858211:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858218:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177252.858221:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.858222:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177252.858240:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.858243:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177252.858245:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177252.858249:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177252.858252:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177252.858255:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177252.858260:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177252.858268:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856355840:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177252.858272:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177252.858275:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858281:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177252.858283:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177252.858286:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177252.858287:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177252.858290:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177252.858293:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856355968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177252.858296:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177252.858298:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177252.858299:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177252.858301:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177252.858305:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7800.
02000000:00000001:0.0:1786177252.858307:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177252.858310:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177252.858313:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177252.858316:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177252.858319:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856355968, portal 10
00000100:00000001:0.0:1786177252.858322:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177252.858323:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:0.0:1786177252.858328:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858335:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177252.858338:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177252.858341:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177252.858343:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856355968, offset 0
00000400:00000200:0.0:1786177252.858347:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177252.858352:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177252.858356:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177252.858359:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177252.858362:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177252.858364:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177252.858367:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177252.858370:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254952:254952:256:4294967295] 192.168.202.110@tcp LPNI seq info [254941:254941:8:4294967295]
00000400:00000200:0.0:1786177252.858374:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177252.858380:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177252.858386:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177252.858390:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177252.858394:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177252.858397:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786177252.858402:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177252.858405:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786177252.858409:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177252.858411:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177252.858415:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177252.858418:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177252.858422:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177252.858423:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177252.858426:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177252.858430:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858438:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177252.858442:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858448:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.858450:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177252.858454:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177252.858457:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177252.858461:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177252.858463:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302148743
00000100:00000001:0.0:1786177252.858466:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302152397
00000100:00000001:0.0:1786177252.858469:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302148743
00000100:00000001:0.0:1786177252.858471:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302152398
00000100:00000001:0.0:1786177252.858473:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177252.858475:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.858477:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177252.858482:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177252/real 1786177252]  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177252.858490:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177252.858492:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177252.858493:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177252.858496:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177252.858498:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177252.858500:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177252.858503:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177252.858533:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858541:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177252.858542:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.858544:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177252.858545:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177252.858549:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858556:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177252.858560:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858567:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177252.858569:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.858571:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177252.858572:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.858574:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177252.858576:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177252.858578:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177252.858582:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177252.858584:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177252.858589:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177252.858598:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856355968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177252.858602:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177252.858605:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858612:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177252.858614:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177252.858616:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177252.858618:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177252.858620:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177252.858623:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856356096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177252.858627:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177252.858629:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177252.858630:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177252.858632:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177252.858636:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177252.858638:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177252.858641:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177252.858644:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177252.858648:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177252.858651:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856356096, portal 4
00000100:00000001:0.0:1786177252.858653:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177252.858654:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:0.0:1786177252.858660:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858668:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177252.858670:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177252.858672:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177252.858675:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856356096, offset 0
00000400:00000200:0.0:1786177252.858679:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177252.858683:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177252.858686:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177252.858689:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177252.858692:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177252.858695:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177252.858697:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177252.858701:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254953:254953:256:4294967295] 192.168.202.110@tcp LPNI seq info [254942:254942:8:4294967295]
00000400:00000200:0.0:1786177252.858705:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177252.858711:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177252.858716:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177252.858720:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177252.858724:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177252.858727:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786177252.858731:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177252.858734:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786177252.858738:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177252.858740:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177252.858744:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177252.858747:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177252.858751:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177252.858753:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177252.858756:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177252.858759:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858767:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177252.858770:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858777:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.858779:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177252.858781:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177252.858784:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177252.858787:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177252.858790:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302148619
00000100:00000001:0.0:1786177252.858792:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302152397
00000100:00000001:0.0:1786177252.858794:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302148619
00000100:00000001:0.0:1786177252.858797:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302152398
00000100:00000001:0.0:1786177252.858799:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177252.858801:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.858803:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177252.858807:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177252/real 1786177252]  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177252.858816:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177252.858818:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177252.858820:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177252.858822:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177252.858824:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177252.858827:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177252.858830:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177252.858839:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858846:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177252.858848:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.858850:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177252.858851:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177252.858855:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858863:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177252.858867:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858875:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177252.858877:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.858879:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177252.858881:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.858882:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177252.858884:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177252.858887:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177252.858891:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177252.858893:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177252.858897:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177252.858907:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856356096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177252.858911:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177252.858914:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858920:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177252.858922:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177252.858924:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177252.858926:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177252.858927:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177252.858930:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856356224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177252.858933:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177252.858935:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177252.858936:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177252.858938:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177252.858943:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786177252.858945:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177252.858948:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177252.858951:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177252.858954:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177252.858957:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856356224, portal 4
00000100:00000001:0.0:1786177252.858960:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177252.858962:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:0.0:1786177252.858967:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.858974:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177252.858976:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177252.858979:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177252.858981:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856356224, offset 0
00000400:00000200:0.0:1786177252.858986:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177252.858990:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177252.858994:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177252.858997:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177252.859000:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177252.859002:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177252.859004:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177252.859008:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254954:254954:256:4294967295] 192.168.202.110@tcp LPNI seq info [254943:254943:8:4294967295]
00000400:00000200:0.0:1786177252.859011:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177252.859018:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177252.859022:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177252.859026:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177252.859030:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177252.859033:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1786177252.859037:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177252.859039:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1786177252.859044:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177252.859046:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177252.859049:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177252.859053:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177252.859056:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177252.859058:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177252.859061:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177252.859065:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.859079:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177252.859082:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.859088:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.859090:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177252.859093:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177252.859096:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177252.859100:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177252.859102:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302148627
00000100:00000001:0.0:1786177252.859104:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302152397
00000100:00000001:0.0:1786177252.859106:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302148627
00000100:00000001:0.0:1786177252.859109:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302152398
00000100:00000001:0.0:1786177252.859111:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177252.859113:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.859115:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177252.859118:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177252/real 1786177252]  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177252.859126:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177252.859127:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177252.859129:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177252.859131:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177252.859133:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177252.859135:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177252.859138:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177252.859146:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.859152:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177252.859155:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.859157:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177252.859158:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177252.859162:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.859168:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177252.859172:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.859179:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177252.859181:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.859183:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177252.859184:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.859186:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177252.859188:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177252.859190:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177252.859193:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177252.859196:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177252.859200:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177252.859208:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856356224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177252.859211:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.859213:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177252.859216:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856355840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.859238:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177252.859241:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177252.859243:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177252.859246:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177252.859247:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177252.859249:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177252.859251:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177252.859253:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:0.0:1786177252.859255:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177252.859256:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177252.859257:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:0.0:1786177252.859259:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177252.859261:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.859263:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177252.859267:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856355968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.859273:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177252.859274:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177252.859275:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7800.
02000000:00000001:0.0:1786177252.859277:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177252.859278:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177252.859279:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177252.859281:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177252.859282:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177252.859284:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177252.859286:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177252.859287:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:0.0:1786177252.859289:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177252.859290:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.859292:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177252.859294:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856356096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.859300:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177252.859302:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177252.859303:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177252.859305:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177252.859306:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177252.859307:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177252.859309:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177252.859310:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786177252.859312:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177252.859313:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177252.859315:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:0.0:1786177252.859317:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177252.859318:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.859319:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177252.859322:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856356224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177252.859328:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177252.859329:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177252.859330:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786177252.859332:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177252.859333:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177252.859334:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177252.859336:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177252.859337:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786177252.859339:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177252.859340:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177252.859341:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:0.0:1786177252.859343:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177252.859344:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.859346:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177252.859350:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177252.859351:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177252.859353:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177252.859355:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177253.432635:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177253.432641:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177253.432648:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177253.432650:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177253.432653:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177253.432654:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177253.432768:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177253.432773:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177253.432783:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177253.880612:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177253.880619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177253.880620:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177253.880634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177253.880637:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177253.880640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177253.880642:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177253.880644:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177253.880646:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177254.456619:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177254.456626:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177254.456631:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177254.456632:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177254.456636:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177254.456639:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177254.456641:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177254.456644:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177254.456647:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177254.456649:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177254.456651:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177254.456653:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177254.456704:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177254.456707:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177254.456710:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177254.904603:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177254.904610:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177254.904612:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177254.904617:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177254.904620:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177254.904634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177254.904637:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177254.904639:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177254.904642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177255.480615:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177255.480628:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177255.480637:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177255.480639:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177255.480642:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177255.480643:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177255.480692:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177255.480695:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177255.480697:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177255.928629:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177255.928636:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177255.928638:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177255.928643:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177255.928646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177255.928650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177255.928653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177255.928655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177255.928658:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177256.504614:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177256.504620:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177256.504626:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177256.504627:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177256.504630:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177256.504632:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177256.504694:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177256.504697:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177256.504700:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177256.696647:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302152397, at 4302152448
00000100:00000010:0.0:1786177256.696663:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000040:1.0:1786177256.696754:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302152398, at 4302152448
00000100:00000010:1.0:1786177256.696760:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134060.
00000100:00000001:2.0:1786177256.952647:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177256.952657:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177256.952659:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177256.952667:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177256.952672:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177256.952676:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177256.952680:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177256.952684:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177256.952687:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177257.528648:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177257.528654:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177257.528659:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177257.528661:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177257.528664:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177257.528665:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177257.528733:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177257.528739:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177257.528742:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177257.976645:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1786177257.976645:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786177257.976656:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:3.0:1786177257.976659:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177257.976660:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.976663:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:2.0:1786177257.976668:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:3.0:1786177257.976669:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177257.976675:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177257.976677:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786177257.976683:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786177257.976688:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7481
00000100:00000001:2.0:1786177257.976689:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177257.976695:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177257.976700:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177257.976701:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177257.976710:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177257.976714:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177257.976717:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177257.976720:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177257.976722:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177257.976723:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177257.976727:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177257.976729:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177257.976731:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177257.976734:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177257.976736:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177257.976738:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177257.976740:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177257.976742:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177257.976744:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177257.976745:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177257.976747:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177257.976749:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177257.976751:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177257.976753:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177257.976756:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177257.976758:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.976761:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177257.976763:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177257.976770:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d180.
00000020:00000040:3.0:1786177257.976774:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177257.976778:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177257.976780:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.976784:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177257.976787:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177257.976793:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177257.976803:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000001:3.0:1786177257.976811:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177257.976821:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177257.976831:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177257.976836:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177257.976841:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085df314e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177257.976866:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177257.976869:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177257.976872:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177257.976874:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177257.976875:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177257.976878:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177257.976881:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7481
00000400:00000200:3.0:1786177257.976884:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177257.976887:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177257.976896:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177257.976900:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177257.976902:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177257.976904:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177257.976906:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177257.976907:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177257.976912:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:0.0:1786177257.976913:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1786177257.976914:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:0.0:1786177257.976918:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1786177257.976918:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177257.976923:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786177257.976926:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177257.976929:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177257.976929:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177257.976934:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177257.976938:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177257.976943:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177257.976948:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177257.976948:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:0.0:1786177257.976951:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177257.976953:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
02000000:00000001:0.0:1786177257.976956:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:3.0:1786177257.976959:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:0.0:1786177257.976961:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177257.976964:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177257.976965:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177257.976968:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:0.0:1786177257.976972:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856356352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177257.976977:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177257.976978:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177257.976982:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177257.976982:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.976986:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:0.0:1786177257.976987:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1786177257.976990:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1786177257.976991:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177257.976997:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177257.976999:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1786177257.977001:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.977004:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177257.977008:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:3.0:1786177257.977009:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e300.
00000020:00000040:3.0:1786177257.977013:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177257.977015:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:0.0:1786177257.977016:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
02000000:00000001:3.0:1786177257.977016:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.977019:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177257.977022:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177257.977023:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177257.977027:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000400:00000010:0.0:1786177257.977028:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000001:3.0:1786177257.977030:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177257.977036:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:0.0:1786177257.977038:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856356352, portal 25
00000100:00000001:0.0:1786177257.977045:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177257.977045:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177257.977049:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297088 : -114547945254528 : ffff97d1ba69d180)
00000100:00000001:3.0:1786177257.977049:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177257.977053:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e3752a60] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177257.977061:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177257.977062:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177257.977066:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177257.977071:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177257.977074:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177257.977077:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177257.977082:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786177257.977083:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1786177257.977100:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7481
00000100:00000040:0.0:1786177257.977102:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177257.977108:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000400:00000200:3.0:1786177257.977109:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000200:0.0:1786177257.977114:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856356352, offset 0
00000400:00000200:3.0:1786177257.977117:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177257.977121:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177257.977125:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177257.977128:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177257.977130:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177257.977134:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177257.977136:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177257.977138:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177257.977140:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000200:0.0:1786177257.977141:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:3.0:1786177257.977142:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177257.977145:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786177257.977147:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177257.977149:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:0.0:1786177257.977153:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:3.0:1786177257.977153:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177257.977155:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177257.977158:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177257.977160:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177257.977161:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:0.0:1786177257.977164:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1786177257.977165:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177257.977169:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177257.977170:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:0.0:1786177257.977171:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:3.0:1786177257.977173:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1786177257.977191:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177257.977194:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:3.0:1786177257.977199:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.977202:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:0.0:1786177257.977204:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254955:254955:256:4294967295] 192.168.202.110@tcp LPNI seq info [254944:254944:8:4294967295]
00000008:00100000:3.0:1786177257.977205:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177257.977210:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.977213:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177257.977215:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c700.
00000020:00000040:3.0:1786177257.977217:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177257.977219:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177257.977220:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177257.977222:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177257.977223:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177257.977226:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177257.977228:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177257.977231:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:3.0:1786177257.977234:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177257.977241:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:0.0:1786177257.977242:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1786177257.977254:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000200:0.0:1786177257.977258:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177257.977259:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177257.977265:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e56458ae] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:0.0:1786177257.977269:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000100:00000001:3.0:1786177257.977271:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177257.977274:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177257.977277:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000800:00000200:0.0:1786177257.977280:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:3.0:1786177257.977280:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177257.977281:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177257.977286:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:0.0:1786177257.977288:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00080000:3.0:1786177257.977290:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7481
00000400:00000200:3.0:1786177257.977297:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000200:0.1:1786177257.977300:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000400:00000200:3.0:1786177257.977305:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000200:0.1:1786177257.977307:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00080000:3.0:1786177257.977309:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177257.977316:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000100:0.0:1786177257.977317:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177257.977322:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000100:00000001:3.0:1786177257.977322:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177257.977327:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:0.0:1786177257.977330:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1786177257.977331:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786177257.977338:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:3.0:1786177257.977344:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:0.0:1786177257.977345:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000020:00000040:3.0:1786177257.977346:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177257.977350:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177257.977352:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177257.977354:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177257.977355:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177257.977358:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000010:0.0:1786177257.977359:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:3.0:1786177257.977360:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177257.977364:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177257.977366:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177257.977368:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:0.0:1786177257.977369:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:3.0:1786177257.977370:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177257.977374:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177257.977375:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:0.0:1786177257.977378:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177257.977379:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177257.977381:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.977383:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177257.977386:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177257.977395:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.977396:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1786177257.977401:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177257.977402:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1786177257.977406:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c000.
00000020:00000040:3.0:1786177257.977411:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177257.977413:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177257.977414:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.977415:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177257.977416:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177257.977418:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177257.977422:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:0.0:1786177257.977425:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.977426:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.977431:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:3.0:1786177257.977436:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177257.977441:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.977445:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:0.0:1786177257.977449:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177257.977450:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.977455:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302157517
00000100:00000040:3.0:1786177257.977458:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006fe8fdb7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1786177257.977462:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134cc0.
00000100:00000001:3.0:1786177257.977472:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.977474:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:3.0:1786177257.977475:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177257.977480:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177257.977481:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177257.977482:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177257.977484:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302157517
00000100:00000001:3.0:1786177257.977485:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177257.977489:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7481
00000100:00000010:0.0:1786177257.977491:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d185134a80.
00000400:00000200:3.0:1786177257.977492:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177257.977497:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1786177257.977499:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00080000:3.0:1786177257.977499:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177257.977504:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177257.977506:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177257.977507:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1786177257.977531:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.977534:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000040:3.0:1786177257.977536:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177257.977540:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00100000:0.0:1786177257.977541:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177257/real 1786177257]  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1786177257.977542:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177257.977545:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177257.977547:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177257.977550:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177257.977554:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177257.977558:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000001:0.0:1786177257.977561:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:3.0:1786177257.977564:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000001:0.0:1786177257.977565:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1786177257.977568:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:0.0:1786177257.977570:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000100:00000040:3.0:1786177257.977571:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177257.977575:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:0.0:1786177257.977576:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000020:00000040:3.0:1786177257.977577:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000400:00000010:0.0:1786177257.977582:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00080000:3.0:1786177257.977582:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177257.977585:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.977588:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:3.0:1786177257.977588:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177257.977590:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177257.977593:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:0.0:1786177257.977596:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177257.977596:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177257.977598:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177257.977602:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.977605:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177257.977610:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ea00.
00000020:00000040:3.0:1786177257.977613:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177257.977616:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:0.0:1786177257.977617:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177257.977618:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177257.977622:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177257.977624:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177257.977625:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177257.977630:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:0.0:1786177257.977634:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177257.977634:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.977638:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177257.977641:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177257.977643:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177257.977647:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177257.977649:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177257.977652:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177257.977656:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1786177257.977657:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053162887] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177257.977675:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.977681:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177257.977681:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7491)
00000100:00000040:0.0:1786177257.977691:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.977699:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177257.977709:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.977711:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177257.977712:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.977715:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177257.977717:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177257.977720:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177257.977725:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177257.977728:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177257.977733:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177257.977742:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856356352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177257.977746:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.977748:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177257.977751:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d180 x1872934856356352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.977758:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177257.977761:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177257.977764:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177257.977767:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177257.977769:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177257.977771:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:0.0:1786177257.977774:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177257.977776:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:0.0:1786177257.977781:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177257.977782:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177257.977785:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d180.
00000100:00000001:0.0:1786177257.977788:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177257.977789:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.977791:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.977799:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177257.977800:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.977803:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177257.977804:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177257.977806:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177257.977810:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.977817:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177257.977818:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177257.977820:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177257.977822:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177257.977824:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177257.977827:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856356480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177257.977831:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177257.977832:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177257.977834:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177257.977836:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177257.977839:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177257.977840:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177257.977844:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177257.977847:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177257.977850:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177257.977853:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856356480, portal 10
00000100:00000001:0.0:1786177257.977855:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177257.977856:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:0.0:1786177257.977863:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.977870:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177257.977872:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177257.977874:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177257.977876:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856356480, offset 0
00000400:00000200:0.0:1786177257.977879:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177257.977883:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177257.977886:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177257.977889:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177257.977891:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177257.977893:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177257.977896:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177257.977898:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254956:254956:256:4294967295] 192.168.202.110@tcp LPNI seq info [254945:254945:8:4294967295]
00000400:00000200:0.0:1786177257.977903:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177257.977908:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177257.977913:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177257.977917:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177257.977921:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177257.977924:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786177257.977928:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177257.977931:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786177257.977934:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177257.977936:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177257.977939:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177257.977942:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177257.977945:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177257.977947:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177257.977949:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177257.977953:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.977960:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177257.977963:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.977969:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.977971:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177257.977974:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177257.977977:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:0.0:1786177257.977980:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177257.977982:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302153738
00000100:00000001:0.0:1786177257.977985:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302157517
00000100:00000001:0.0:1786177257.977987:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302153738
00000100:00000001:0.0:1786177257.977990:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302157517
00000100:00000001:0.0:1786177257.977992:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177257.977994:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.977995:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177257.977999:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177257/real 1786177257]  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177257.978014:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177257.978015:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177257.978017:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177257.978019:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177257.978021:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177257.978023:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177257.978026:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177257.978033:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978040:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177257.978041:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978043:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177257.978045:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177257.978048:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978055:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177257.978058:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978064:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177257.978067:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978069:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177257.978070:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978071:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177257.978073:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177257.978076:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177257.978078:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177257.978081:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177257.978085:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177257.978092:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856356480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177257.978096:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177257.978100:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978106:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177257.978107:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177257.978109:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177257.978111:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177257.978113:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177257.978115:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856356608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177257.978118:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177257.978120:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177257.978122:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177257.978123:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177257.978127:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786177257.978129:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177257.978132:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177257.978134:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177257.978137:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177257.978140:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856356608, portal 10
00000100:00000001:0.0:1786177257.978141:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177257.978143:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:0.0:1786177257.978149:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978155:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177257.978157:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177257.978159:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177257.978161:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856356608, offset 0
00000400:00000200:0.0:1786177257.978166:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177257.978169:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177257.978173:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177257.978187:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177257.978189:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177257.978192:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177257.978194:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177257.978197:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254957:254957:256:4294967295] 192.168.202.110@tcp LPNI seq info [254946:254946:8:4294967295]
00000400:00000200:0.0:1786177257.978201:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177257.978207:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177257.978211:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177257.978214:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177257.978218:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177257.978221:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786177257.978225:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177257.978227:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786177257.978231:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177257.978232:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177257.978235:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177257.978239:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177257.978241:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177257.978243:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177257.978245:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177257.978249:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978256:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177257.978259:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978265:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.978267:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177257.978270:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177257.978272:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:0.0:1786177257.978275:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177257.978278:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302153863
00000100:00000001:0.0:1786177257.978280:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302157517
00000100:00000001:0.0:1786177257.978282:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302153863
00000100:00000001:0.0:1786177257.978284:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302157517
00000100:00000001:0.0:1786177257.978286:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177257.978288:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.978290:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177257.978293:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177257/real 1786177257]  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177257.978300:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177257.978302:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177257.978303:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177257.978305:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177257.978307:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177257.978309:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177257.978312:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177257.978320:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978327:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177257.978328:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978330:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177257.978331:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177257.978335:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978342:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177257.978346:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978352:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177257.978355:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978357:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177257.978358:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978359:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177257.978361:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177257.978364:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177257.978366:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177257.978368:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177257.978371:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177257.978379:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856356608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:0.0:1786177257.978383:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177257.978386:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978392:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177257.978394:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177257.978396:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177257.978398:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177257.978400:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177257.978402:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856356736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177257.978405:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177257.978407:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177257.978409:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177257.978410:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177257.978413:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a7800.
02000000:00000001:0.0:1786177257.978415:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177257.978418:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177257.978421:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177257.978424:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177257.978427:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856356736, portal 4
00000100:00000001:0.0:1786177257.978429:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177257.978430:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:0.0:1786177257.978436:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978442:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177257.978444:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177257.978446:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177257.978448:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856356736, offset 0
00000400:00000200:0.0:1786177257.978452:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177257.978455:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177257.978458:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177257.978461:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177257.978463:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177257.978465:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177257.978467:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177257.978470:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254958:254958:256:4294967295] 192.168.202.110@tcp LPNI seq info [254947:254947:8:4294967295]
00000400:00000200:0.0:1786177257.978474:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177257.978479:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177257.978484:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177257.978487:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177257.978491:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177257.978493:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786177257.978497:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177257.978499:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786177257.978503:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177257.978505:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177257.978507:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177257.978530:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177257.978533:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177257.978535:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177257.978538:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177257.978541:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978554:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177257.978557:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978563:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.978564:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177257.978567:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177257.978569:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:0.0:1786177257.978572:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177257.978574:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302153739
00000100:00000001:0.0:1786177257.978577:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302157517
00000100:00000001:0.0:1786177257.978579:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302153739
00000100:00000001:0.0:1786177257.978581:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302157517
00000100:00000001:0.0:1786177257.978583:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177257.978585:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.978587:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177257.978590:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177257/real 1786177257]  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177257.978598:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177257.978599:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177257.978601:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177257.978603:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177257.978605:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177257.978607:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177257.978610:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177257.978617:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978623:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177257.978625:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978627:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177257.978628:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177257.978631:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978638:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177257.978641:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978649:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177257.978651:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978652:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177257.978654:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978655:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177257.978657:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177257.978659:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177257.978662:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177257.978665:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177257.978669:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177257.978677:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856356736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177257.978681:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1786177257.978684:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978691:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177257.978692:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177257.978695:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1786177257.978697:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177257.978698:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177257.978702:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856356864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177257.978704:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177257.978706:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177257.978708:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177257.978709:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1786177257.978712:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177257.978714:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177257.978717:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:0.0:1786177257.978720:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d18381c058.
00000400:00000010:0.0:1786177257.978723:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4330.
00000100:00000200:0.0:1786177257.978726:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856356864, portal 4
00000100:00000001:0.0:1786177257.978727:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177257.978729:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:0.0:1786177257.978734:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978740:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177257.978742:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:0.0:1786177257.978744:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831c4c38.
00000100:00000200:0.0:1786177257.978746:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856356864, offset 0
00000400:00000200:0.0:1786177257.978750:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:0.0:1786177257.978754:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177257.978757:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:0.0:1786177257.978759:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1786177257.978761:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:0.0:1786177257.978764:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177257.978766:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177257.978769:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254959:254959:256:4294967295] 192.168.202.110@tcp LPNI seq info [254948:254948:8:4294967295]
00000400:00000200:0.0:1786177257.978773:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177257.978778:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1786177257.978782:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177257.978785:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1aad45b00.
00000800:00000200:0.0:1786177257.978789:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.0:1786177257.978792:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1786177257.978796:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:0.1:1786177257.978798:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1786177257.978802:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:0.0:1786177257.978804:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1aad45b00.
00000400:00000200:0.0:1786177257.978806:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1786177257.978809:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1786177257.978812:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4c38
00000400:00000010:0.0:1786177257.978814:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4c38.
00000100:00000001:0.0:1786177257.978816:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1786177257.978819:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978826:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177257.978829:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978835:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.978837:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177257.978839:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177257.978842:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:0.0:1786177257.978845:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177257.978847:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302153747
00000100:00000001:0.0:1786177257.978849:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302157517
00000100:00000001:0.0:1786177257.978851:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302153747
00000100:00000001:0.0:1786177257.978853:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302157517
00000100:00000001:0.0:1786177257.978855:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177257.978857:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.978859:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1786177257.978862:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177257/real 1786177257]  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1786177257.978869:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1786177257.978871:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1786177257.978872:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d18381c058.
00000400:00000200:0.0:1786177257.978874:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831c4330
00000400:00000010:0.0:1786177257.978875:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831c4330.
00000100:00000001:0.0:1786177257.978878:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1786177257.978880:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1786177257.978888:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978894:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1786177257.978895:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978897:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177257.978899:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177257.978902:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978908:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177257.978912:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978919:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177257.978921:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978923:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177257.978924:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978925:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1786177257.978927:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1786177257.978930:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1786177257.978933:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:0.0:1786177257.978935:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1786177257.978939:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1786177257.978952:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856356864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:0.0:1786177257.978955:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978957:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177257.978960:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856356480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.978966:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177257.978968:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177257.978970:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a4000.
02000000:00000001:0.0:1786177257.978972:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177257.978974:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177257.978975:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177257.978977:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177257.978979:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:0.0:1786177257.978982:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177257.978983:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177257.978985:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:0.0:1786177257.978988:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177257.978989:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.978990:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177257.978994:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856356608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.979000:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177257.979001:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177257.979003:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a6000.
02000000:00000001:0.0:1786177257.979005:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177257.979006:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177257.979007:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:0.0:1786177257.979009:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177257.979011:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:0.0:1786177257.979013:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177257.979014:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177257.979015:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:0.0:1786177257.979018:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177257.979019:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.979021:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177257.979024:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856356736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.979030:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177257.979031:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177257.979032:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a7800.
02000000:00000001:0.0:1786177257.979034:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177257.979035:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177257.979036:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177257.979038:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177257.979039:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:0.0:1786177257.979042:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177257.979043:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177257.979044:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:0.0:1786177257.979046:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177257.979047:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.979049:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177257.979052:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856356864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177257.979058:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177257.979059:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1786177257.979060:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1885a5800.
02000000:00000001:0.0:1786177257.979063:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1786177257.979064:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:0.0:1786177257.979065:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:0.0:1786177257.979066:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:0.0:1786177257.979068:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:0.0:1786177257.979070:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1786177257.979071:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1786177257.979072:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:0.0:1786177257.979074:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1786177257.979075:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.979077:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177257.979080:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177257.979082:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177257.979084:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177257.979086:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177258.552625:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177258.552639:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177258.552644:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177258.552646:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177258.552650:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177258.552651:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177258.552742:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177258.552748:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177258.552751:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177259.000674:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177259.000685:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177259.000688:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177259.000695:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177259.000700:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177259.000705:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177259.000708:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177259.000711:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177259.000715:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177259.576680:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177259.576687:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177259.576692:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177259.576693:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177259.576695:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177259.576696:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177259.576762:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177259.576767:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177259.576772:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.0:1786177259.896696:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:3.0:1786177259.896706:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:3.0:1786177259.896712:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:1.0:1786177259.896938:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1786177259.896945:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786177259.896964:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1786177259.896971:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786177259.896974:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177260.024637:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177260.024648:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177260.024651:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177260.024658:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177260.024662:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177260.024668:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177260.024671:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177260.024675:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177260.024678:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177260.600691:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177260.600700:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177260.600703:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177260.600706:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177260.600712:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177260.600714:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177260.600716:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177260.600718:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177260.600721:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177260.600723:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177260.600723:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177260.600725:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177260.600726:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177260.600726:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177260.600727:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177260.600728:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177260.600730:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177260.600731:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177260.600790:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177260.600800:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177260.600803:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177261.048680:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177261.048691:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177261.048693:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177261.048701:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177261.048704:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177261.048709:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177261.048713:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177261.048717:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177261.048720:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177261.624681:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177261.624690:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177261.624697:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177261.624698:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177261.624702:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177261.624703:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177261.624772:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177261.624775:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177261.624778:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177261.816663:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302157517, at 4302157568
00000100:00000040:1.0:1786177261.816663:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302157517, at 4302157568
00000100:00000010:1.0:1786177261.816681:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134a80.
00000100:00000010:0.0:1786177261.816684:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d185134cc0.
00000100:00000001:2.0:1786177262.072666:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177262.072678:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177262.072680:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177262.072686:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177262.072689:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177262.072693:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177262.072695:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177262.072697:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177262.072700:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177262.648647:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177262.648657:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177262.648663:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177262.648664:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177262.648666:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177262.648667:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177262.648741:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177262.648746:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177262.648750:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786177263.096646:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786177263.096646:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177263.096656:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786177263.096663:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177263.096666:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.096667:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177263.096672:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1786177263.096674:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177263.096679:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786177263.096681:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177263.096686:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177263.096689:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7486
00000100:00000001:2.0:1786177263.096692:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177263.096696:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1786177263.096700:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177263.096701:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177263.096710:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177263.096715:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177263.096719:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.096723:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177263.096726:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.096728:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177263.096733:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177263.096736:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177263.096739:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177263.096742:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177263.096745:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177263.096748:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177263.096752:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.096754:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177263.096755:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.096757:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177263.096760:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177263.096763:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177263.096766:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177263.096769:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177263.096773:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177263.096776:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.096778:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177263.096780:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177263.096795:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41ad80.
00000020:00000040:0.0:1786177263.096798:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177263.096803:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177263.096805:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.096809:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177263.096812:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177263.096823:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177263.096832:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5800.
00000100:00000001:0.0:1786177263.096841:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177263.096852:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177263.096863:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177263.096868:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177263.096873:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034e2c676] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177263.096895:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177263.096899:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177263.096902:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177263.096904:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177263.096906:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177263.096909:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177263.096912:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7486
00000400:00000200:0.0:1786177263.096916:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177263.096920:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177263.096923:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177263.096925:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177263.096927:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177263.096928:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177263.096933:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177263.096934:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177263.096936:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177263.096938:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1786177263.096940:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177263.096943:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177263.096945:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177263.096948:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177263.096950:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177263.096950:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177263.096955:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177263.096963:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.096968:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177263.096968:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177263.096972:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177263.096973:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.096978:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177263.096980:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1786177263.096981:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177263.096986:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1786177263.096986:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177263.096991:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177263.096996:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856356992:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177263.096999:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.097000:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177263.097004:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177263.097006:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177263.097009:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177263.097011:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177263.097012:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1786177263.097016:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.097019:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000010:3.0:1786177263.097019:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177263.097022:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1786177263.097024:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177263.097029:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177263.097029:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177263.097033:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000001:0.0:1786177263.097034:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177263.097040:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41aa00.
00000400:00000010:3.0:1786177263.097040:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000020:00000040:0.0:1786177263.097043:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177263.097047:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:3.0:1786177263.097049:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856356992, portal 25
02000000:00000001:0.0:1786177263.097050:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.097054:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1786177263.097055:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177263.097057:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761666432 : -114547947885184 : ffff97d1ba41ad80)
02000000:00000001:0.0:1786177263.097058:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177263.097062:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177263.097064:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786177263.097068:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7800.
00000100:00000001:3.0:1786177263.097074:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1786177263.097075:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177263.097078:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177263.097083:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00100000:0.0:1786177263.097086:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1786177263.097087:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856356992, offset 0
00000400:00000200:3.0:1786177263.097097:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097098:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177263.097102:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177263.097108:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da0255a4] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177263.097110:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177263.097119:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:0.0:1786177263.097120:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177263.097135:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177263.097139:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177263.097142:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1786177263.097142:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:0.0:1786177263.097146:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177263.097150:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177263.097150:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:0.0:1786177263.097156:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7486
00000400:00000200:3.0:1786177263.097160:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1786177263.097163:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177263.097167:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:0.0:1786177263.097171:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177263.097175:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1786177263.097177:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254960:254960:256:4294967295] 192.168.202.110@tcp LPNI seq info [254949:254949:8:4294967295]
00000100:00000040:0.0:1786177263.097181:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097186:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177263.097189:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177263.097190:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177263.097194:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177263.097197:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177263.097199:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177263.097201:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177263.097203:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177263.097207:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177263.097211:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000001:0.0:1786177263.097212:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177263.097218:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097222:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177263.097227:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:3.0:1786177263.097227:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097232:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177263.097236:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000800:00000010:3.0:1786177263.097237:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000100:00080000:0.0:1786177263.097239:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097243:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177263.097244:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177263.097247:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:3.0:1786177263.097249:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:0.0:1786177263.097250:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.097254:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000800:00000200:3.0:1786177263.097257:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000008:00100000:0.0:1786177263.097258:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177263.097262:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.097266:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177263.097270:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000010:0.0:1786177263.097272:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419500.
00000020:00000040:0.0:1786177263.097277:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000800:00000200:3.1:1786177263.097277:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
02000000:00000001:0.0:1786177263.097281:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177263.097283:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1786177263.097287:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097288:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177263.097291:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177263.097294:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1786177263.097294:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
02000000:00000010:0.0:1786177263.097299:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6000.
00000100:00000001:0.0:1786177263.097304:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177263.097304:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:0.0:1786177263.097310:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177263.097315:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1786177263.097325:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177263.097327:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000100:00000001:0.0:1786177263.097328:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177263.097332:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000040:0.0:1786177263.097336:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b6718a9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.097340:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177263.097347:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177263.097349:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177263.097354:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177263.097362:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177263.097364:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177263.097365:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1786177263.097366:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177263.097371:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:3.0:1786177263.097373:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177263.097376:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7486
00000400:00000200:0.0:1786177263.097383:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177263.097387:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177263.097390:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177263.097393:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097394:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177263.097396:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097398:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177263.097400:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177263.097401:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177263.097402:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177263.097404:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177263.097406:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177263.097408:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177263.097411:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097413:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177263.097415:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097416:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177263.097418:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177263.097420:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097423:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177263.097425:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177263.097427:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177263.097429:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.097431:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177263.097434:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177263.097438:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.097443:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177263.097451:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a680.
00000020:00000040:0.0:1786177263.097456:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177263.097460:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177263.097462:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.097465:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177263.097465:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177263.097467:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177263.097472:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.097474:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:0.0:1786177263.097478:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4000.
00000100:00000001:0.0:1786177263.097483:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177263.097488:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177263.097492:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.097503:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177263.097507:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177263.097510:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177263.097537:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000911ee1e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177263.097546:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177263.097549:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177263.097553:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177263.097555:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177263.097557:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177263.097559:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177263.097563:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7486
00000100:00000001:3.0:1786177263.097568:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302162638
00000400:00000200:0.0:1786177263.097570:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177263.097579:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177263.097582:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177263.097586:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000010:3.0:1786177263.097586:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37480.
00000100:00000001:0.0:1786177263.097590:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177263.097595:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097598:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177263.097600:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000020:00000001:0.0:1786177263.097602:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177263.097605:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177263.097607:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177263.097610:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177263.097614:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:3.0:1786177263.097614:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302162638
00000020:00000001:0.0:1786177263.097623:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000010:3.0:1786177263.097626:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad376c0.
00000100:00000040:0.0:1786177263.097631:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097636:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177263.097637:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:0.0:1786177263.097642:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177263.097646:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177263.097647:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177263.097651:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:3.0:1786177263.097660:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177263.097661:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177263.097669:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.097671:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
02000000:00000001:0.0:1786177263.097672:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177263.097678:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177263.097682:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.097686:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177263.097689:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00100000:3.0:1786177263.097690:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177263/real 1786177263]  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1786177263.097695:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.097702:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177263.097712:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a300.
00000020:00000040:0.0:1786177263.097720:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000400:00000001:3.0:1786177263.097723:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:0.0:1786177263.097726:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000001:3.0:1786177263.097728:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:0.0:1786177263.097729:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177263.097734:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1786177263.097735:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
02000000:00000001:0.0:1786177263.097737:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177263.097741:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177263.097744:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
02000000:00000010:0.0:1786177263.097747:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000400:00000010:3.0:1786177263.097750:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177263.097753:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.097758:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00100000:0.0:1786177263.097763:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1786177263.097770:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177263.097788:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177263.097795:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177263.097807:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b0d3569] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177263.097809:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177263.097843:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177263.097852:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7496)
00000100:00000001:3.0:1786177263.097854:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177263.097860:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.097864:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177263.097866:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177263.097874:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.097883:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177263.097889:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.097897:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177263.097900:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.097902:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177263.097903:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.097906:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177263.097908:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177263.097912:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177263.097917:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177263.097919:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177263.097924:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177263.097933:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856356992:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177263.097938:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.097941:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177263.097945:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41ad80 x1872934856356992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.097952:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177263.097955:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177263.097958:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177263.097963:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177263.097965:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177263.097967:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177263.097969:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177263.097971:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5800.
02000000:00000001:3.0:1786177263.097975:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177263.097977:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177263.097979:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41ad80.
00000100:00000001:3.0:1786177263.097982:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177263.097983:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.097986:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.097993:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177263.097995:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.097997:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177263.097999:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177263.098001:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177263.098007:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098013:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177263.098015:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177263.098018:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177263.098021:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177263.098024:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177263.098027:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856357120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177263.098030:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177263.098034:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177263.098036:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177263.098038:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177263.098042:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177263.098045:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177263.098049:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177263.098052:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177263.098056:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177263.098061:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856357120, portal 10
00000100:00000001:3.0:1786177263.098063:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177263.098064:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761665536 : -114547947886080 : ffff97d1ba41aa00)
00000100:00000040:3.0:1786177263.098071:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098078:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177263.098080:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177263.098083:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177263.098086:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856357120, offset 0
00000400:00000200:3.0:1786177263.098091:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177263.098098:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177263.098104:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177263.098109:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177263.098113:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177263.098116:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177263.098119:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177263.098123:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254961:254961:256:4294967295] 192.168.202.110@tcp LPNI seq info [254950:254950:8:4294967295]
00000400:00000200:3.0:1786177263.098143:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177263.098152:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177263.098158:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177263.098163:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177263.098170:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177263.098173:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786177263.098179:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177263.098182:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1786177263.098187:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177263.098189:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177263.098192:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177263.098196:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177263.098200:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177263.098202:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177263.098205:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177263.098208:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098216:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177263.098220:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098226:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.098228:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177263.098231:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177263.098234:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177263.098238:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177263.098240:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302158858
00000100:00000001:3.0:1786177263.098242:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302162638
00000100:00000001:3.0:1786177263.098244:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302158858
00000100:00000001:3.0:1786177263.098246:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302162638
00000100:00000001:3.0:1786177263.098248:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177263.098250:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.098252:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177263.098256:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177263/real 1786177263]  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177263.098263:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177263.098264:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177263.098266:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177263.098268:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177263.098270:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177263.098272:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177263.098276:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177263.098283:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098290:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177263.098291:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.098293:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177263.098294:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177263.098297:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098303:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177263.098307:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098313:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177263.098315:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.098317:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177263.098318:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.098320:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177263.098322:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177263.098325:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177263.098328:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177263.098331:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177263.098334:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177263.098342:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856357120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177263.098345:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177263.098349:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098356:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177263.098357:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177263.098359:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177263.098361:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177263.098363:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177263.098366:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856357248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177263.098369:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177263.098371:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177263.098373:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177263.098375:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177263.098379:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177263.098381:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177263.098384:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177263.098386:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177263.098389:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177263.098391:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856357248, portal 10
00000100:00000001:3.0:1786177263.098393:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177263.098394:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761660160 : -114547947891456 : ffff97d1ba419500)
00000100:00000040:3.0:1786177263.098400:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098411:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177263.098413:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177263.098415:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177263.098417:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856357248, offset 0
00000400:00000200:3.0:1786177263.098421:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177263.098425:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177263.098428:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177263.098431:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177263.098433:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177263.098436:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177263.098437:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177263.098440:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254962:254962:256:4294967295] 192.168.202.110@tcp LPNI seq info [254951:254951:8:4294967295]
00000400:00000200:3.0:1786177263.098444:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177263.098450:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177263.098455:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177263.098458:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177263.098462:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177263.098465:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786177263.098469:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177263.098471:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1786177263.098474:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177263.098476:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177263.098479:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177263.098482:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177263.098484:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177263.098486:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177263.098488:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177263.098492:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098500:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177263.098502:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098540:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.098542:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177263.098546:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177263.098548:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177263.098551:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177263.098553:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302158984
00000100:00000001:3.0:1786177263.098555:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302162638
00000100:00000001:3.0:1786177263.098557:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302158984
00000100:00000001:3.0:1786177263.098559:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302162638
00000100:00000001:3.0:1786177263.098561:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177263.098563:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.098565:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177263.098568:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177263/real 1786177263]  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177263.098576:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177263.098577:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177263.098579:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177263.098581:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177263.098582:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177263.098584:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177263.098588:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177263.098596:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098602:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177263.098604:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.098605:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177263.098607:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177263.098610:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098616:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177263.098619:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098627:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177263.098629:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.098630:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177263.098632:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.098633:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177263.098635:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177263.098638:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177263.098641:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177263.098643:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177263.098647:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177263.098654:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856357248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177263.098658:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177263.098662:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098668:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177263.098669:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177263.098671:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177263.098673:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177263.098674:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177263.098677:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856357376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177263.098679:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177263.098681:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177263.098683:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177263.098685:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177263.098689:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177263.098691:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177263.098694:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177263.098697:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177263.098700:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177263.098703:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856357376, portal 4
00000100:00000001:3.0:1786177263.098705:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177263.098706:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761664640 : -114547947886976 : ffff97d1ba41a680)
00000100:00000040:3.0:1786177263.098711:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098718:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177263.098720:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177263.098722:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177263.098724:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856357376, offset 0
00000400:00000200:3.0:1786177263.098728:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177263.098732:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177263.098735:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177263.098738:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177263.098740:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177263.098742:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177263.098745:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177263.098748:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254963:254963:256:4294967295] 192.168.202.110@tcp LPNI seq info [254952:254952:8:4294967295]
00000400:00000200:3.0:1786177263.098751:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177263.098757:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177263.098761:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177263.098764:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177263.098767:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177263.098770:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786177263.098773:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177263.098776:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1786177263.098779:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177263.098782:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177263.098784:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177263.098788:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177263.098790:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177263.098792:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177263.098794:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177263.098798:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098805:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177263.098808:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098815:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.098816:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177263.098819:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177263.098822:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177263.098825:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177263.098827:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302158859
00000100:00000001:3.0:1786177263.098829:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302162638
00000100:00000001:3.0:1786177263.098831:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302158859
00000100:00000001:3.0:1786177263.098833:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302162638
00000100:00000001:3.0:1786177263.098835:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177263.098837:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.098839:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177263.098843:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177263/real 1786177263]  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177263.098851:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177263.098852:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177263.098853:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177263.098856:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177263.098857:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177263.098860:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177263.098863:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177263.098872:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098878:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177263.098879:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.098881:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177263.098882:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177263.098886:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098892:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177263.098896:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098903:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177263.098905:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.098906:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177263.098907:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.098909:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177263.098910:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177263.098913:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177263.098916:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177263.098918:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177263.098922:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177263.098930:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856357376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177263.098934:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177263.098938:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098944:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177263.098946:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177263.098948:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177263.098950:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177263.098952:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177263.098955:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856357504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177263.098958:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177263.098960:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177263.098961:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177263.098963:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177263.098967:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177263.098969:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177263.098972:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177263.098974:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177263.098977:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177263.098980:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856357504, portal 4
00000100:00000001:3.0:1786177263.098981:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177263.098983:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761663744 : -114547947887872 : ffff97d1ba41a300)
00000100:00000040:3.0:1786177263.098988:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.098993:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177263.098995:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177263.098997:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177263.099000:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856357504, offset 0
00000400:00000200:3.0:1786177263.099004:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177263.099008:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177263.099011:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177263.099013:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177263.099016:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177263.099018:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177263.099020:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177263.099023:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254964:254964:256:4294967295] 192.168.202.110@tcp LPNI seq info [254953:254953:8:4294967295]
00000400:00000200:3.0:1786177263.099027:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177263.099033:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177263.099037:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177263.099040:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177263.099044:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177263.099047:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1786177263.099051:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177263.099053:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1786177263.099057:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177263.099059:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177263.099062:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177263.099065:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177263.099068:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177263.099070:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177263.099072:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177263.099075:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.099083:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177263.099086:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.099093:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.099094:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177263.099097:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177263.099100:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177263.099103:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177263.099105:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302158867
00000100:00000001:3.0:1786177263.099107:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302162638
00000100:00000001:3.0:1786177263.099109:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302158867
00000100:00000001:3.0:1786177263.099111:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302162638
00000100:00000001:3.0:1786177263.099113:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177263.099115:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.099117:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177263.099121:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177263/real 1786177263]  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177263.099152:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177263.099153:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177263.099155:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177263.099157:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177263.099158:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177263.099160:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177263.099164:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177263.099172:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.099178:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177263.099179:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.099181:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177263.099182:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177263.099186:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.099192:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177263.099196:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.099202:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177263.099204:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.099205:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177263.099207:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.099208:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177263.099210:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177263.099212:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177263.099216:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177263.099218:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177263.099221:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177263.099229:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856357504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177263.099232:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.099235:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177263.099237:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41aa00 x1872934856357120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.099244:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177263.099245:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177263.099248:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177263.099250:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177263.099251:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177263.099252:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177263.099254:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177263.099255:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7800.
02000000:00000001:3.0:1786177263.099258:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177263.099259:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177263.099260:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41aa00.
00000100:00000001:3.0:1786177263.099263:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177263.099264:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.099265:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177263.099269:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419500 x1872934856357248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.099275:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177263.099276:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177263.099277:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177263.099279:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177263.099280:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177263.099282:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177263.099283:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177263.099285:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6000.
02000000:00000001:3.0:1786177263.099287:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177263.099288:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177263.099289:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419500.
00000100:00000001:3.0:1786177263.099291:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177263.099292:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.099294:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177263.099297:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a680 x1872934856357376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.099302:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177263.099304:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177263.099305:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177263.099307:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177263.099308:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177263.099309:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177263.099310:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177263.099312:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4000.
02000000:00000001:3.0:1786177263.099313:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177263.099314:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177263.099315:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a680.
00000100:00000001:3.0:1786177263.099317:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177263.099318:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.099320:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177263.099323:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a300 x1872934856357504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177263.099329:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177263.099330:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177263.099331:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177263.099333:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177263.099334:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177263.099335:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177263.099336:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177263.099338:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:3.0:1786177263.099339:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177263.099340:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177263.099341:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a300.
00000100:00000001:3.0:1786177263.099344:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177263.099345:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.099347:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177263.099350:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177263.099351:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177263.099353:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177263.099355:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177263.672667:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177263.672675:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177263.672681:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177263.672684:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177263.672687:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177263.672690:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177263.672691:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177263.672695:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177263.672697:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177263.672700:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177263.672702:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177263.672703:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177263.672773:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177263.672778:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177263.672782:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177264.121259:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177264.121269:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177264.121272:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177264.121280:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177264.121285:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177264.121289:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177264.121292:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177264.121295:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177264.121299:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177264.696624:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177264.696630:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177264.696635:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177264.696637:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177264.696641:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177264.696642:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177264.696700:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177264.696703:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177264.696705:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177265.144638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177265.144644:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177265.144646:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177265.144652:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177265.144655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177265.144659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177265.144661:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177265.144664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177265.144667:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177265.720636:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177265.720642:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177265.720647:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177265.720649:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177265.720652:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177265.720653:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177265.720712:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177265.720715:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177265.720718:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177266.168646:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177266.168652:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177266.168654:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177266.168659:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177266.168663:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177266.168666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177266.168669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177266.168671:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177266.168674:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177266.744686:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177266.744692:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177266.744696:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177266.744697:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177266.744699:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177266.744700:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177266.744782:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177266.744790:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177266.744796:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177266.936643:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302162638, at 4302162688
00000100:00000010:1.0:1786177266.936660:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad376c0.
00000100:00000040:0.0:1786177266.936671:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302162638, at 4302162688
00000100:00000010:0.0:1786177266.936687:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37480.
00000100:00000001:2.0:1786177267.192595:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177267.192602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177267.192603:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177267.192607:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177267.192610:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177267.192612:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177267.192614:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177267.192616:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177267.192618:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177267.768634:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177267.768641:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177267.768648:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177267.768649:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177267.768653:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177267.768654:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177267.768714:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177267.768718:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177267.768721:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786177268.216625:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786177268.216625:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177268.216631:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786177268.216633:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177268.216635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.216636:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177268.216641:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1786177268.216642:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177268.216646:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786177268.216648:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177268.216651:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177268.216654:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7491
00000100:00000001:2.0:1786177268.216656:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177268.216660:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177268.216663:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177268.216664:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177268.216671:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177268.216673:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177268.216674:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216677:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177268.216679:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216681:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177268.216683:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177268.216685:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177268.216688:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177268.216691:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177268.216692:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177268.216694:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177268.216696:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216698:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177268.216699:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216700:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177268.216701:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177268.216703:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216705:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177268.216707:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177268.216709:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177268.216711:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.216712:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177268.216713:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177268.216721:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419c00.
00000020:00000040:0.0:1786177268.216724:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177268.216727:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177268.216729:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.216731:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177268.216734:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177268.216739:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177268.216746:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000100:00000001:0.0:1786177268.216753:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177268.216763:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177268.216773:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177268.216778:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177268.216783:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cf890c1c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177268.216811:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177268.216815:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177268.216818:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177268.216820:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177268.216821:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177268.216824:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177268.216827:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7491
00000400:00000200:0.0:1786177268.216830:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177268.216833:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177268.216835:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177268.216838:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216840:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177268.216841:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216844:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177268.216846:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177268.216847:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177268.216849:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177268.216851:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177268.216853:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177268.216854:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177268.216857:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216859:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177268.216861:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216863:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177268.216865:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177268.216867:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216869:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177268.216871:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177268.216876:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177268.216877:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.216879:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177268.216883:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177268.216886:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.216888:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177268.216892:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41bb80.
00000020:00000040:0.0:1786177268.216894:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177268.216897:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177268.216898:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.216900:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177268.216901:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177268.216903:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177268.216907:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000100:00000001:0.0:1786177268.216910:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177268.216914:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177268.216920:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177268.216921:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177268.216923:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9983a05] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177268.216928:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177268.216930:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177268.216932:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177268.216933:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177268.216934:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177268.216936:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177268.216937:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7491
00000400:00000200:0.0:1786177268.216939:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177268.216941:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177268.216943:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177268.216944:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216946:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177268.216947:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216948:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177268.216950:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177268.216951:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177268.216952:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177268.216953:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177268.216954:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177268.216955:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177268.216957:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216958:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177268.216959:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216960:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177268.216961:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786177268.216962:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177268.216963:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177268.216965:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177268.216966:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177268.216967:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.216969:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177268.216970:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177268.216971:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.216972:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177268.216974:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418700.
00000020:00000040:0.0:1786177268.216976:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177268.216977:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177268.216978:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.216979:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177268.216980:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177268.216981:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177268.216982:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:0.0:1786177268.216984:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177268.216986:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177268.216994:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177268.216995:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177268.216997:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bf66f89d] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177268.217003:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177268.217005:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177268.217008:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177268.217009:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177268.217010:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177268.217013:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177268.217015:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7491
00000400:00000200:0.0:1786177268.217018:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177268.217020:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177268.217022:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177268.217025:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.217027:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177268.217029:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.217030:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177268.217032:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177268.217033:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177268.217035:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177268.217037:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177268.217039:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177268.217041:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177268.217044:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.217046:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177268.217048:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.217050:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177268.217052:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177268.217055:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177268.217056:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177268.217058:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177268.217059:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177268.217061:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.217062:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177268.217064:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177268.217065:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.217066:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177268.217069:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418000.
00000020:00000040:0.0:1786177268.217070:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177268.217071:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177268.217072:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.217073:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177268.217074:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177268.217103:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177268.217112:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000100:00000001:2.0:1786177268.217113:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1786177268.217116:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177268.217118:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177268.217122:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00100000:0.0:1786177268.217126:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1786177268.217131:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177268.217145:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177268.217147:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177268.217149:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177268.217150:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177268.217156:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1786177268.217159:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002e10e4f5] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177268.217159:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177268.217165:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177268.217173:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856357632:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177268.217176:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177268.217178:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:0.0:1786177268.217180:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:2.0:1786177268.217182:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177268.217186:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177268.217187:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177268.217191:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:2.0:1786177268.217191:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:0.0:1786177268.217194:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177268.217197:0:62275:0:(import.c:485:import_select_connection()) Process entered
02000000:00000010:2.0:1786177268.217198:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3000.
00000100:00080000:0.0:1786177268.217201:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7491
02000000:00000001:2.0:1786177268.217202:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177268.217208:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177268.217212:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177268.217216:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000010:2.0:1786177268.217218:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7318.
00000100:00080000:0.0:1786177268.217219:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177268.217225:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.217228:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1786177268.217228:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a26e8.
00000100:00000040:0.0:1786177268.217232:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.217236:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:2.0:1786177268.217237:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856357632, portal 25
00000020:00000001:0.0:1786177268.217238:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177268.217242:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:2.0:1786177268.217242:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177268.217245:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761661952 : -114547947889664 : ffff97d1ba419c00)
00000020:00000001:0.0:1786177268.217246:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177268.217250:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177268.217253:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177268.217255:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177268.217256:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786177268.217263:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.217267:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1786177268.217270:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177268.217271:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.217276:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:2.0:1786177268.217276:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177268.217280:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2088.
00000020:00000040:0.0:1786177268.217281:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000200:2.0:1786177268.217285:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856357632, offset 0
00000100:00080000:0.0:1786177268.217287:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177268.217291:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177268.217294:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1786177268.217295:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786177268.217298:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177268.217301:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.217304:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:2.0:1786177268.217304:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:0.0:1786177268.217307:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177268.217310:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177268.217313:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177268.217313:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177268.217320:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000010:0.0:1786177268.217321:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b800.
00000020:00000040:0.0:1786177268.217325:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000400:00000200:2.0:1786177268.217326:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000001:0.0:1786177268.217331:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1786177268.217332:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1786177268.217334:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177268.217337:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1786177268.217338:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177268.217342:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177268.217345:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177268.217345:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254965:254965:256:4294967295] 192.168.202.110@tcp LPNI seq info [254954:254954:8:4294967295]
02000000:00000010:0.0:1786177268.217352:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000400:00000200:2.0:1786177268.217353:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177268.217357:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177268.217366:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1786177268.217369:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177268.217380:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177268.217381:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177268.217385:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1786177268.217387:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000100:00000040:0.0:1786177268.217391:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000192ee9a0] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1786177268.217397:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177268.217405:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177268.217407:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177268.217414:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7501)
00000100:00000001:2.0:1786177268.217427:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177268.217433:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177268.217435:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302167757
00000100:00000010:2.0:1786177268.217440:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25360.
00000100:00000001:2.0:1786177268.217446:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177268.217451:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302167757
00000100:00000010:2.0:1786177268.217454:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25d20.
00000100:00000001:2.0:1786177268.217457:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786177268.217460:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177268.217462:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177268.217464:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177268.217468:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177268.217475:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177268.217476:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177268.217478:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177268.217480:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177268.217482:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177268.217484:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856357760:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177268.217487:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177268.217489:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177268.217490:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177268.217492:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177268.217494:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab0000.
02000000:00000001:2.0:1786177268.217496:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177268.217499:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177268.217502:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a73c8.
00000400:00000010:2.0:1786177268.217505:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177268.217507:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856357760, portal 10
00000100:00000001:2.0:1786177268.217526:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177268.217528:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761670016 : -114547947881600 : ffff97d1ba41bb80)
00000100:00000040:2.0:1786177268.217534:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177268.217540:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177268.217542:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177268.217545:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2880.
00000100:00000200:2.0:1786177268.217547:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856357760, offset 0
00000400:00000200:2.0:1786177268.217551:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177268.217554:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177268.217557:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177268.217560:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177268.217562:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177268.217564:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177268.217566:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177268.217569:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254966:254966:255:4294967295] 192.168.202.110@tcp LPNI seq info [254955:254955:7:4294967295]
00000400:00000200:2.0:1786177268.217573:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177268.217578:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177268.217583:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177268.217587:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef800.
00000800:00000200:2.0:1786177268.217590:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177268.217594:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177268.217596:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177268.217599:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:2.0:1786177268.217602:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177268.217604:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302163978
00000100:00000001:2.0:1786177268.217606:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302167757
00000100:00000001:2.0:1786177268.217609:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302163978
00000100:00000001:2.0:1786177268.217611:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302167757
00000100:00000001:2.0:1786177268.217613:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177268.217615:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177268.217617:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177268.217619:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177268.217626:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177268.217627:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177268.217629:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177268.217631:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177268.217633:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177268.217635:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856357888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177268.217638:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177268.217640:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177268.217641:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177268.217643:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177268.217645:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1c00.
02000000:00000001:2.0:1786177268.217647:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177268.217649:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177268.217651:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177268.217654:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177268.217657:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856357888, portal 10
00000100:00000001:2.0:1786177268.217658:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177268.217660:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761656576 : -114547947895040 : ffff97d1ba418700)
00000100:00000040:2.0:1786177268.217664:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177268.217670:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177268.217672:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177268.217674:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2440.
00000100:00000200:2.0:1786177268.217676:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856357888, offset 0
00000400:00000200:2.0:1786177268.217680:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177268.217682:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177268.217685:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177268.217688:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177268.217690:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177268.217692:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177268.217694:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177268.217697:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254967:254967:254:4294967295] 192.168.202.110@tcp LPNI seq info [254956:254956:6:4294967295]
00000400:00000200:2.0:1786177268.217700:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177268.217706:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177268.217710:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177268.217713:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef000.
00000800:00000200:2.0:1786177268.217715:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177268.217718:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177268.217721:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177268.217723:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:2.0:1786177268.217726:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177268.217728:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302164103
00000100:00000001:2.0:1786177268.217730:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302167757
00000100:00000001:2.0:1786177268.217732:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302164103
00000100:00000001:2.0:1786177268.217734:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302167757
00000100:00000001:2.0:1786177268.217736:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177268.217737:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177268.217739:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177268.217742:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177268.217747:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177268.217749:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1786177268.217751:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177268.217752:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177268.217753:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177268.217756:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177268.217762:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177268.217764:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177268.217765:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177268.217767:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177268.217769:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177268.217771:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856358016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177268.217774:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177268.217775:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177268.217776:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177268.217778:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177268.217780:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1000.
02000000:00000001:2.0:1786177268.217782:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177268.217785:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177268.217787:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7ec8.
00000400:00000010:2.0:1786177268.217790:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177268.217792:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856358016, portal 4
00000100:00000001:2.0:1786177268.217794:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177268.217795:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761654784 : -114547947896832 : ffff97d1ba418000)
00000100:00000040:2.0:1786177268.217799:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177268.217805:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177268.217807:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177268.217809:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2ee0.
00000100:00000200:2.0:1786177268.217810:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856358016, offset 0
00000400:00000200:2.0:1786177268.217814:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177268.217816:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177268.217819:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177268.217821:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177268.217823:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177268.217825:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177268.217827:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177268.217830:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254968:254968:253:4294967295] 192.168.202.110@tcp LPNI seq info [254957:254957:5:4294967295]
00000400:00000200:2.0:1786177268.217834:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177268.217839:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177268.217843:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177268.217846:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efe00.
00000800:00000200:2.0:1786177268.217849:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177268.217852:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177268.217854:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177268.217857:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:2.0:1786177268.217860:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177268.217861:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302163978
00000100:00000001:2.0:1786177268.217863:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302167757
00000100:00000001:2.0:1786177268.217866:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302163978
00000100:00000001:2.0:1786177268.217867:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302167757
00000100:00000001:2.0:1786177268.217869:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177268.217870:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177268.217872:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177268.217875:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177268.217881:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177268.217882:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177268.217884:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177268.217885:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177268.217887:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177268.217889:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856358144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177268.217892:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177268.217893:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177268.217894:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177268.217896:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177268.217898:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177268.217900:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177268.217902:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177268.217904:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7dc0.
00000400:00000010:2.0:1786177268.217907:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a22a8.
00000100:00000200:2.0:1786177268.217909:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856358144, portal 4
00000100:00000001:2.0:1786177268.217910:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177268.217912:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761669120 : -114547947882496 : ffff97d1ba41b800)
00000100:00000040:2.0:1786177268.217917:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177268.217922:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177268.217924:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177268.217926:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2550.
00000100:00000200:2.0:1786177268.217928:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856358144, offset 0
00000400:00000200:2.0:1786177268.217931:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177268.217934:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177268.217937:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177268.217939:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177268.217941:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177268.217942:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177268.217945:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177268.217947:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254969:254969:252:4294967295] 192.168.202.110@tcp LPNI seq info [254958:254958:4:4294967295]
00000400:00000200:2.0:1786177268.217951:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177268.217957:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177268.217960:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177268.217963:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef200.
00000800:00000200:2.0:1786177268.217966:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177268.217969:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177268.217971:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177268.217973:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:2.0:1786177268.217976:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177268.217977:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302163986
00000100:00000001:2.0:1786177268.217979:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302167757
00000100:00000001:2.0:1786177268.217981:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302163986
00000100:00000001:2.0:1786177268.217983:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302167757
00000100:00000001:2.0:1786177268.217985:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177268.217987:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177268.217989:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177268.217991:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177268.217993:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177268.217994:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1786177268.217996:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177268.217997:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177268.217999:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177268.218001:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177268.792625:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177268.792630:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177268.792635:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177268.792636:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177268.792639:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177268.792641:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177268.792701:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177268.792706:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177268.792709:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177269.240619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177269.240628:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177269.240630:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177269.240636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177269.240639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177269.240644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177269.240647:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177269.240649:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177269.240653:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177269.816632:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177269.816639:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177269.816646:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177269.816648:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177269.816651:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177269.816652:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177269.816846:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177269.816849:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177269.816852:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177270.264622:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177270.264630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177270.264632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177270.264637:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177270.264641:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177270.264644:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177270.264648:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177270.264651:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177270.264653:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177270.841015:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177270.841023:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177270.841026:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177270.841029:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177270.841033:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177270.841035:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177270.841037:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177270.841040:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177270.841043:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177270.841045:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177270.841046:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177270.841047:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177270.841049:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177270.841064:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177270.841066:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177270.841068:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177270.841071:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177270.841073:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177270.841173:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177270.841176:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177270.841179:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177271.288715:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177271.288722:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177271.288725:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177271.288730:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177271.288733:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177271.288737:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.288740:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177271.288742:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.288745:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1786177271.289287:0:2379:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1786177271.289325:0:2379:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1786177271.289336:0:2379:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177271.289340:0:2379:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1786177271.289344:0:2379:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177271.289349:0:2379:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1786177271.289354:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177271.289358:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:1.0:1786177271.289364:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1786177271.289369:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177271.289375:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2088
00000400:00000010:1.0:1786177271.289377:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2088.
00000100:00000001:1.0:1786177271.289383:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177271.289390:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177271.289420:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177271.289425:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177271.289434:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177271.289436:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1786177271.289445:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177271.289447:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef800.
00000400:00000200:1.0:1786177271.289450:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1786177271.289454:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177271.289456:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2880
00000400:00000010:1.0:1786177271.289458:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2880.
00000100:00000001:1.0:1786177271.289461:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177271.289466:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177271.289473:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177271.289476:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177271.289483:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177271.289485:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1786177271.289490:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177271.289492:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef000.
00000400:00000200:1.0:1786177271.289495:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1786177271.289498:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177271.289501:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2440
00000400:00000010:1.0:1786177271.289502:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2440.
00000100:00000001:1.0:1786177271.289505:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177271.289509:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177271.289553:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177271.289556:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177271.289563:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177271.289565:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1786177271.289570:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177271.289572:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efe00.
00000400:00000200:1.0:1786177271.289575:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1786177271.289578:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177271.289581:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2ee0
00000400:00000010:1.0:1786177271.289582:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2ee0.
00000100:00000001:1.0:1786177271.289584:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177271.289588:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177271.289595:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177271.289598:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177271.289605:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177271.289606:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1786177271.289610:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177271.289612:0:2379:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef200.
00000400:00000200:1.0:1786177271.289615:0:2379:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1786177271.289618:0:2379:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177271.289620:0:2379:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2550
00000400:00000010:1.0:1786177271.289622:0:2379:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2550.
00000100:00000001:1.0:1786177271.289624:0:2379:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177271.289628:0:2379:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177271.289635:0:2379:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177271.289638:0:2379:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177293 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177271.289644:0:2379:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177271.289645:0:2379:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177271.289697:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177271.289700:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177271.289702:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177271.289708:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177268/real 1786177271]  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177271.289719:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177271.289721:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177271.289724:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7ec8.
00000400:00000200:2.0:1786177271.289727:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177271.289729:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177271.289731:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177271.289735:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177271.289743:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.289749:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177271.289750:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.289753:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177271.289755:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177271.289758:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.289765:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177271.289769:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.289777:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177271.289779:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.289781:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177271.289782:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.289785:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177271.289787:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177271.289791:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177271.289795:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177271.289798:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177271.289803:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177271.289811:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856358016:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177271.289815:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177271.289818:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177268/real 1786177271]  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177271.289825:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177271.289826:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177271.289828:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7dc0.
00000400:00000200:2.0:1786177271.289831:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a22a8
00000400:00000010:2.0:1786177271.289832:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a22a8.
00000100:00000001:2.0:1786177271.289834:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177271.289838:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177271.289845:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.289851:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177271.289852:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.289854:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177271.289855:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177271.289858:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.289863:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177271.289867:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.289872:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177271.289874:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.289876:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177271.289877:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.289878:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177271.289880:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177271.289882:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177271.289886:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177271.289893:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177271.289898:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177271.289905:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856358144:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177271.289909:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177271.289913:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177268/real 1786177271]  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177271.289920:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177271.289922:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177271.289924:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7318.
00000400:00000200:2.0:1786177271.289926:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a26e8
00000400:00000010:2.0:1786177271.289929:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a26e8.
00000100:00000001:2.0:1786177271.289931:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177271.289935:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177271.289943:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.289950:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177271.289952:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.289954:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177271.289955:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177271.289959:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.289964:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177271.289967:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.289973:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177271.289975:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.289976:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177271.289977:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.289979:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177271.289981:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177271.289983:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177271.289986:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177271.289988:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177271.289991:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177271.289998:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856357632:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177271.290002:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177271.290005:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177268/real 1786177271]  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177271.290013:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177271.290014:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177271.290016:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a73c8.
00000400:00000200:2.0:1786177271.290018:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177271.290020:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177271.290022:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177271.290026:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177271.290034:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.290040:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177271.290041:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290043:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177271.290044:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177271.290579:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.290587:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177271.290591:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.290597:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177271.290599:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290600:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177271.290601:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290603:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177271.290605:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177271.290607:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177271.290610:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177271.290612:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177271.290615:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177271.290622:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856357760:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177271.290626:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177271.290629:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177268/real 1786177271]  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177271.290636:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177271.290637:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177271.290639:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177271.290642:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177271.290644:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177271.290645:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177271.290648:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177271.290654:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.290660:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177271.290661:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290663:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177271.290664:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177271.290667:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.290672:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177271.290675:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.290681:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177271.290683:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290684:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177271.290685:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290686:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177271.290688:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177271.290690:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177271.290693:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177271.290695:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177271.290699:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177271.290706:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856357888:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177271.290710:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290713:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177271.290715:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418000 x1872934856358016/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.290721:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177271.290724:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177271.290727:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1000.
02000000:00000001:2.0:1786177271.290731:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177271.290733:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177271.290735:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177271.290737:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177271.290739:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:2.0:1786177271.290742:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177271.290744:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177271.290746:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418000.
00000100:00000001:2.0:1786177271.290751:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177271.290752:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290753:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177271.290756:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b800 x1872934856358144/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.290762:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177271.290763:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177271.290765:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177271.290767:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177271.290768:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177271.290769:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177271.290771:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177271.290772:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000001:2.0:1786177271.290774:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177271.290776:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177271.290777:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b800.
00000100:00000001:2.0:1786177271.290780:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177271.290781:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290783:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177271.290786:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419c00 x1872934856357632/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.290792:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177271.290793:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177271.290795:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177271.290797:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177271.290798:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177271.290799:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1786177271.290801:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177271.290802:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:2.0:1786177271.290804:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177271.290805:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177271.290806:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419c00.
00000100:00000001:2.0:1786177271.290808:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177271.290809:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290811:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177271.290814:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41bb80 x1872934856357760/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.290820:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177271.290821:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177271.290822:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab0000.
02000000:00000001:2.0:1786177271.290825:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177271.290825:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177271.290827:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177271.290828:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177271.290830:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:2.0:1786177271.290832:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177271.290833:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177271.290834:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41bb80.
00000100:00000001:2.0:1786177271.290837:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177271.290838:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290839:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177271.290842:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418700 x1872934856357888/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177293 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177271.290848:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177271.290850:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177271.290851:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:2.0:1786177271.290853:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177271.290854:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177271.290855:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177271.290857:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177271.290858:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:2.0:1786177271.290860:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177271.290861:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177271.290862:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418700.
00000100:00000001:2.0:1786177271.290864:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177271.290866:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290868:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177271.290875:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177271.290877:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177271.290879:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177271.290881:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177271.864676:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177271.864683:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177271.864689:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177271.864690:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177271.864693:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177271.864694:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177271.865460:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177271.865464:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177271.865467:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177272.056647:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302167757, at 4302167808
00000100:00000010:1.0:1786177272.056663:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25d20.
00000100:00000040:0.0:1786177272.056787:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302167757, at 4302167808
00000100:00000010:0.0:1786177272.056793:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25360.
00000100:00000001:2.0:1786177272.312749:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177272.312756:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177272.312759:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177272.312764:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177272.312767:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177272.312770:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177272.312773:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177272.312775:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177272.312778:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177272.888638:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177272.888645:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177272.888651:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177272.888652:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177272.888654:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177272.888655:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177272.888725:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177272.888729:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177272.888732:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177273.336766:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177273.336774:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177273.336776:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177273.336781:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177273.336785:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177273.336789:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.336791:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177273.336794:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.336796:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177273.337727:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177273.337736:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177273.337738:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177273.337743:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177273.337747:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177273.337755:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7496
00000400:00000200:3.0:1786177273.337765:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177273.337772:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177273.337776:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177273.337779:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.337782:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177273.337785:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.337787:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177273.337791:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177273.337793:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177273.337796:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177273.337800:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177273.337802:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177273.337805:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177273.337808:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.337810:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177273.337812:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.337814:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177273.337817:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177273.337820:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177273.337823:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177273.337826:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177273.337829:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177273.337832:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.337835:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177273.337836:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177273.337845:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d500.
00000020:00000040:3.0:1786177273.337848:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177273.337852:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177273.337854:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.337858:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177273.337861:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177273.337865:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177273.337872:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786177273.337879:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177273.337889:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177273.337899:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177273.337904:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177273.337909:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7c36393] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177273.337933:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177273.337937:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177273.337940:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177273.337941:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177273.337943:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177273.337945:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177273.337947:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7496
00000400:00000200:3.0:1786177273.337951:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177273.337953:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177273.337955:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177273.337958:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.337960:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177273.337962:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.337963:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177273.337966:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177273.337967:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177273.337968:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177273.337970:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786177273.337972:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786177273.337974:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177273.337977:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.337978:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177273.337980:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.337982:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177273.337984:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786177273.337987:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177273.337989:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177273.337991:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177273.337996:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177273.337998:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.338000:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177273.338003:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177273.338006:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.338008:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177273.338012:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f100.
00000020:00000040:3.0:1786177273.338014:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177273.338016:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177273.338017:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.338019:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177273.338020:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177273.338022:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177273.338052:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:3.0:1786177273.338057:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177273.338062:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177273.338069:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177273.338071:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177273.338075:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1ef9f45] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177273.338082:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177273.338085:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177273.338089:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177273.338090:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177273.338092:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177273.338095:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177273.338097:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7496
00000400:00000200:3.0:1786177273.338100:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177273.338103:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177273.338105:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177273.338108:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338110:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177273.338111:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338113:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177273.338115:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177273.338116:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177273.338118:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177273.338120:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786177273.338121:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177273.338123:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177273.338126:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338127:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177273.338129:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338131:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177273.338134:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786177273.338136:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338139:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177273.338140:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177273.338143:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177273.338145:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.338146:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177273.338148:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177273.338150:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.338152:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177273.338156:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f480.
00000020:00000040:3.0:1786177273.338158:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177273.338160:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177273.338161:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.338163:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177273.338164:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177273.338166:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177273.338170:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:3.0:1786177273.338173:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177273.338177:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177273.338184:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177273.338186:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177273.338189:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092aa1aa7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177273.338196:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177273.338199:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177273.338202:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177273.338203:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177273.338205:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177273.338208:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177273.338211:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7496
00000400:00000200:3.0:1786177273.338214:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177273.338220:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177273.338223:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177273.338226:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338227:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177273.338229:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338231:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177273.338233:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177273.338234:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177273.338236:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177273.338238:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177273.338239:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177273.338241:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177273.338244:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338245:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177273.338247:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338248:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177273.338250:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177273.338253:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338255:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177273.338256:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177273.338258:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177273.338260:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.338262:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177273.338264:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177273.338267:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.338268:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177273.338271:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f800.
00000020:00000040:3.0:1786177273.338273:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177273.338275:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177273.338277:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.338278:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177273.338279:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177273.338281:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177273.338284:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786177273.338287:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177273.338290:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177273.338297:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177273.338299:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177273.338302:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4d339d9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177273.338308:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177273.338311:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177273.338313:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177273.338315:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177273.338316:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177273.338319:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177273.338321:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7496
00000400:00000200:3.0:1786177273.338323:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177273.338325:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177273.338327:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177273.338329:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338331:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177273.338333:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338335:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177273.338337:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177273.338338:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177273.338340:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177273.338342:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177273.338343:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177273.338345:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177273.338348:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338350:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177273.338351:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338353:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177273.338355:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786177273.338357:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177273.338359:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177273.338360:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177273.338362:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177273.338364:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.338366:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177273.338367:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177273.338369:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.338370:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177273.338373:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ca80.
00000020:00000040:3.0:1786177273.338375:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177273.338376:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177273.338377:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177273.338379:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177273.338380:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177273.338381:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177273.338385:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:3.0:1786177273.338387:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177273.338391:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177273.338397:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177273.338399:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177273.338402:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ffb0865] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177273.338418:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177273.338422:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7506)
00000100:00000001:2.0:1786177273.339703:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177273.339710:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177273.339712:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177273.339721:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.339733:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177273.339735:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177273.339739:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177273.339742:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177273.339745:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177273.339748:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856358272:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177273.339752:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177273.339756:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177273.339759:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177273.339761:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177273.339769:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1c00.
02000000:00000001:2.0:1786177273.339771:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177273.339778:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177273.339794:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177273.339803:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177273.339809:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856358272, portal 25
00000100:00000001:2.0:1786177273.339812:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177273.339814:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000100:00000040:2.0:1786177273.339820:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.339827:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177273.339830:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177273.339832:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177273.339836:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856358272, offset 0
00000400:00000200:2.0:1786177273.339842:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177273.339850:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177273.339856:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177273.339870:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177273.339874:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177273.339877:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177273.339880:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177273.339884:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254970:254970:256:4294967295] 192.168.202.110@tcp LPNI seq info [254959:254959:8:4294967295]
00000400:00000200:2.0:1786177273.339888:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177273.339898:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177273.339912:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177273.339917:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177273.339924:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177273.339927:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1786177273.339933:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177273.339936:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1786177273.339940:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177273.339943:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177273.339951:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177273.339955:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177273.339960:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177273.339962:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177273.339966:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177273.339970:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.339981:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177273.339985:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.339998:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177273.339999:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177273.340005:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177273.340009:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177273.340011:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302172880
00000100:00000010:2.0:1786177273.340017:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f250c0.
00000100:00000001:2.0:1786177273.340024:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177273.340233:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302172880
00000100:00000010:2.0:1786177273.340238:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25480.
00000100:00000001:2.0:1786177273.340242:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786177273.340245:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177273.340248:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177273.340251:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177273.340256:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177273/real 1786177273]  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177273.340266:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177273.340267:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177273.340269:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177273.340272:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177273.340274:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177273.340276:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177273.340279:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177273.340288:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.340295:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177273.340296:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.340298:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177273.340300:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177273.340304:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.340310:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177273.340314:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.340321:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177273.340324:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.340325:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177273.340327:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.340329:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177273.340331:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177273.340334:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177273.340338:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177273.340340:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177273.340344:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177273.340352:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856358272:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177273.340356:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177273.340360:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.340365:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177273.340367:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177273.340369:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177273.340372:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177273.340373:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177273.340376:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856358400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177273.340378:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177273.340380:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177273.340382:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177273.340383:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177273.340387:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab0000.
02000000:00000001:2.0:1786177273.340388:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177273.340391:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177273.340394:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177273.340396:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177273.340400:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856358400, portal 10
00000100:00000001:2.0:1786177273.340402:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177273.340403:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:2.0:1786177273.340410:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.340415:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177273.340417:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177273.340419:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177273.340421:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856358400, offset 0
00000400:00000200:2.0:1786177273.340425:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177273.340429:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177273.340432:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177273.340435:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177273.340437:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177273.340440:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177273.340442:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177273.340445:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254971:254971:256:4294967295] 192.168.202.110@tcp LPNI seq info [254960:254960:8:4294967295]
00000400:00000200:2.0:1786177273.340448:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177273.340455:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177273.340459:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177273.340462:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177273.340466:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177273.340469:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1786177273.340472:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177273.340475:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1786177273.340478:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177273.340480:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177273.340483:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177273.340486:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177273.340489:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177273.340491:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177273.340494:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177273.340497:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.340648:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177273.340653:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.340660:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177273.340662:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177273.340666:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177273.340670:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:2.0:1786177273.340673:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177273.340675:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302169100
00000100:00000001:2.0:1786177273.340677:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302172880
00000100:00000001:2.0:1786177273.340679:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302169100
00000100:00000001:2.0:1786177273.340681:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302172880
00000100:00000001:2.0:1786177273.340683:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177273.340685:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177273.340687:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177273.340691:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177273/real 1786177273]  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177273.340698:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177273.340699:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177273.340701:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177273.340703:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177273.340705:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177273.340707:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177273.340710:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177273.340718:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.340724:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177273.340726:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.340728:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177273.340729:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177273.340732:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.340738:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177273.340742:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.340748:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177273.340750:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.340752:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177273.340753:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.340755:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177273.340757:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177273.340759:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177273.340763:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177273.340765:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177273.340768:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177273.340776:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856358400:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177273.340780:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177273.340973:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.340981:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177273.340982:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177273.340985:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177273.340986:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177273.340988:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177273.340991:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856358528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177273.340994:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177273.340996:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177273.340998:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177273.340999:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177273.341003:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177273.341005:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177273.341008:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177273.341011:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177273.341014:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177273.341017:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856358528, portal 10
00000100:00000001:2.0:1786177273.341018:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177273.341020:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:2.0:1786177273.341115:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.341123:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177273.341126:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177273.341129:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177273.341133:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856358528, offset 0
00000400:00000200:2.0:1786177273.341137:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177273.341142:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177273.341147:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177273.341151:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177273.341154:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177273.341252:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177273.341257:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177273.341261:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254972:254972:256:4294967295] 192.168.202.110@tcp LPNI seq info [254961:254961:8:4294967295]
00000400:00000200:2.0:1786177273.341265:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177273.341272:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177273.341277:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177273.341281:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177273.341285:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177273.341288:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1786177273.341292:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177273.341294:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1786177273.341297:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177273.341299:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177273.341303:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177273.341307:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177273.341310:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177273.341312:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177273.341314:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177273.341318:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.341326:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177273.341328:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.341334:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177273.341336:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177273.341339:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177273.341342:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:2.0:1786177273.341345:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177273.341347:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302169226
00000100:00000001:2.0:1786177273.341349:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302172880
00000100:00000001:2.0:1786177273.341351:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302169226
00000100:00000001:2.0:1786177273.341353:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302172880
00000100:00000001:2.0:1786177273.341355:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177273.341357:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177273.341359:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177273.341363:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177273/real 1786177273]  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177273.341370:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177273.341372:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177273.341374:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177273.341376:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177273.341377:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177273.341379:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177273.341382:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177273.341390:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.341396:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177273.341398:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.341399:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177273.341401:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177273.341404:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.341411:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177273.341415:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.341421:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177273.341423:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.341425:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177273.341426:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.341428:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177273.341430:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177273.341432:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177273.341435:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177273.341437:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177273.341441:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177273.342384:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856358528:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177273.342389:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177273.342393:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.342400:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177273.342402:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177273.342405:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177273.342407:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177273.342409:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177273.342412:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856358656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177273.342416:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177273.342418:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177273.342420:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177273.342422:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177273.342428:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177273.342430:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177273.342435:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177273.342439:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177273.342443:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177273.342622:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856358656, portal 4
00000100:00000001:2.0:1786177273.342625:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177273.342626:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
00000100:00000040:2.0:1786177273.342633:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.342638:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177273.342640:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177273.342643:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177273.342647:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856358656, offset 0
00000400:00000200:2.0:1786177273.342651:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177273.342656:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177273.342661:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177273.342664:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177273.342666:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177273.342669:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177273.342671:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177273.342675:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254973:254973:256:4294967295] 192.168.202.110@tcp LPNI seq info [254962:254962:8:4294967295]
00000400:00000200:2.0:1786177273.342679:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177273.342761:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177273.342767:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177273.342770:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177273.342775:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177273.342778:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1786177273.343019:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177273.343022:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1786177273.343044:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177273.343047:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177273.343051:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177273.343055:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177273.343060:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177273.343062:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177273.343065:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177273.343069:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.343080:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177273.343083:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.343089:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177273.343091:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177273.343095:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177273.343099:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:2.0:1786177273.343102:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177273.343105:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302169103
00000100:00000001:2.0:1786177273.343109:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302172880
00000100:00000001:2.0:1786177273.343111:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302169103
00000100:00000001:2.0:1786177273.343113:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302172880
00000100:00000001:2.0:1786177273.343115:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177273.343118:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177273.343120:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177273.343124:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177273/real 1786177273]  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177273.343133:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177273.343135:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177273.343137:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177273.343140:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177273.343142:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177273.343144:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177273.343147:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177273.343155:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.343288:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177273.343291:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.343293:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177273.343295:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177273.343300:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.343307:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177273.343311:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.343318:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177273.343321:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.343322:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177273.343324:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.343326:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177273.343328:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177273.343331:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177273.343335:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177273.343338:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177273.343341:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177273.343349:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856358656:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177273.343354:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177273.343358:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.343364:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177273.343366:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177273.343369:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177273.343370:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177273.343373:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177273.343375:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856358784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177273.343378:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177273.343380:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177273.343382:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177273.343384:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177273.343388:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1000.
02000000:00000001:2.0:1786177273.343390:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177273.343394:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177273.343397:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177273.343400:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177273.343404:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856358784, portal 4
00000100:00000001:2.0:1786177273.343406:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177273.343407:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:2.0:1786177273.343413:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.343419:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177273.343422:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177273.343424:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177273.343427:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856358784, offset 0
00000400:00000200:2.0:1786177273.343431:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177273.343435:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177273.343438:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177273.343441:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177273.343444:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177273.343779:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177273.343914:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177273.343920:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254974:254974:256:4294967295] 192.168.202.110@tcp LPNI seq info [254963:254963:8:4294967295]
00000400:00000200:2.0:1786177273.343925:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177273.343933:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177273.343938:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177273.343942:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177273.343946:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177273.343949:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1786177273.343953:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177273.343955:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1786177273.343959:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177273.343961:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177273.343965:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177273.343969:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177273.343972:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177273.343974:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177273.343977:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177273.343980:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.343993:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177273.343996:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.344003:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177273.344004:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177273.344009:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177273.344012:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:2.0:1786177273.344015:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177273.344017:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302169112
00000100:00000001:2.0:1786177273.344020:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302172880
00000100:00000001:2.0:1786177273.344022:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302169112
00000100:00000001:2.0:1786177273.344024:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302172880
00000100:00000001:2.0:1786177273.344038:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177273.344041:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177273.344043:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177273.344047:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177273/real 1786177273]  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177273.344056:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177273.344057:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177273.344059:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177273.344062:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177273.344063:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177273.344066:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177273.344069:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177273.344077:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.344083:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177273.344085:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.344087:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177273.344088:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177273.344092:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.344099:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177273.344102:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.344108:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177273.344111:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.344112:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177273.344114:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.344116:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177273.344118:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177273.344121:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177273.344124:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177273.344126:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177273.344130:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177273.344137:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856358784:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177273.344142:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.344144:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177273.344148:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856358272/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.344154:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177273.344157:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177273.344159:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:2.0:1786177273.344162:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177273.344280:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177273.344282:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1786177273.344285:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177273.344287:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:2.0:1786177273.344291:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177273.344293:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177273.344295:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:2.0:1786177273.344299:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177273.344300:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.344302:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177273.344307:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856358400/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.344313:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177273.344315:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177273.344317:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab0000.
02000000:00000001:2.0:1786177273.344320:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177273.344321:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177273.344322:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177273.344324:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177273.344325:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:2.0:1786177273.344327:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177273.344328:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177273.344329:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:2.0:1786177273.344332:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177273.344333:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.344335:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177273.344339:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856358528/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.344345:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177273.344346:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177273.344347:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177273.344350:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177273.344351:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177273.344352:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177273.344354:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177273.344355:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:2.0:1786177273.344357:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177273.344358:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177273.344359:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:2.0:1786177273.344362:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177273.344363:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.344364:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177273.344367:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856358656/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.344373:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177273.344374:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177273.344375:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177273.344377:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177273.344378:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177273.344380:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177273.344382:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177273.344383:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:2.0:1786177273.344385:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177273.344386:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177273.344387:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000001:2.0:1786177273.344389:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177273.344391:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.344392:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177273.344395:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856358784/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177273.344401:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177273.344402:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177273.344404:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1000.
02000000:00000001:2.0:1786177273.344406:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177273.344407:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177273.344408:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177273.344410:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177273.344411:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:2.0:1786177273.344413:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177273.344414:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177273.344416:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:2.0:1786177273.344418:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177273.344419:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.344422:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177273.344429:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177273.344432:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177273.344434:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177273.344436:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177273.912607:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177273.912614:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177273.912619:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177273.912621:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177273.912624:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177273.912627:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177273.912629:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177273.912632:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177273.912636:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177273.912638:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177273.912639:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177273.912642:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177273.912813:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177273.912816:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177273.912819:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177274.360633:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177274.360641:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177274.360643:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177274.360648:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177274.360652:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177274.360656:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177274.360659:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177274.360662:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177274.360665:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177274.936661:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177274.936667:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177274.936874:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177274.936876:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177274.936880:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177274.936881:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177274.936998:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177274.937002:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177274.937006:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177275.384627:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177275.384634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177275.384636:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177275.384642:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177275.384645:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177275.384648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177275.384651:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177275.384654:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177275.384657:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177275.960784:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177275.961209:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177275.961217:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177275.961218:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177275.961222:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177275.961223:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177275.962961:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177275.962966:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177275.962970:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177276.408634:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177276.408643:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177276.408645:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177276.408651:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177276.408655:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177276.408659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177276.408662:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177276.408664:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177276.408667:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177276.984760:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177276.984766:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177276.984771:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177276.984773:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177276.984776:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177276.984777:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177276.984848:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177276.984852:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177276.984854:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177277.176611:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302172880, at 4302172928
00000100:00000010:0.0:1786177277.176628:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f250c0.
00000100:00000040:1.0:1786177277.176709:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302172880, at 4302172928
00000100:00000010:1.0:1786177277.176715:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25480.
00000100:00000001:2.0:1786177277.432614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177277.432620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177277.432622:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177277.432679:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177277.432683:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177277.432686:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177277.432689:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177277.432691:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177277.432695:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177278.008625:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177278.008632:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177278.008639:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177278.008640:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177278.008643:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177278.008644:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177278.008711:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177278.008716:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177278.008719:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177278.456614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177278.456621:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177278.456623:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177278.456628:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177278.456632:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177278.456636:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.456638:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177278.456641:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.456643:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177278.456776:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177278.456782:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177278.456785:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177278.456788:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177278.456792:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177278.456799:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7501
00000400:00000200:0.0:1786177278.456806:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177278.456813:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177278.456816:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177278.456819:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.456822:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177278.456824:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.456827:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177278.456831:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177278.456833:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177278.456836:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177278.456839:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177278.456841:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177278.456844:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177278.456847:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.456849:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177278.456850:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.456852:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177278.456855:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177278.456858:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177278.456860:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177278.456863:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177278.456866:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177278.456868:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.456870:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177278.456872:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177278.456880:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418380.
00000020:00000040:0.0:1786177278.456883:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177278.456887:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177278.456889:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.456892:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177278.456894:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177278.456898:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177278.456905:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
00000100:00000001:0.0:1786177278.456910:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177278.456918:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177278.456929:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177278.456933:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177278.456938:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e6b901d7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177278.456963:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177278.456967:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177278.456970:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177278.456971:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177278.456972:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177278.456975:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786177278.457011:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177278.457015:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177278.457017:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177278.457022:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.457029:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177278.457031:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177278.457034:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177278.457037:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177278.457039:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177278.457042:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856358912:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177278.457046:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177278.457049:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177278.457051:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177278.457053:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177278.457058:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177278.457060:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177278.457065:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177278.457069:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177278.457075:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177278.457081:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856358912, portal 25
00000100:00000001:2.0:1786177278.457083:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177278.457085:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761655680 : -114547947895936 : ffff97d1ba418380)
00000100:00000040:2.0:1786177278.457090:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.457097:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177278.457099:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177278.457102:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177278.457106:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856358912, offset 0
00000400:00000200:2.0:1786177278.457112:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177278.457119:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177278.457124:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177278.457130:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177278.457133:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177278.457137:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177278.457140:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177278.457143:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254975:254975:256:4294967295] 192.168.202.110@tcp LPNI seq info [254964:254964:8:4294967295]
00000400:00000200:2.0:1786177278.457147:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177278.457157:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177278.457163:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177278.457167:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177278.457173:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177278.457177:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1786177278.457183:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177278.457186:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1786177278.457190:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177278.457193:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177278.457200:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177278.457205:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177278.457209:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177278.457211:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177278.457215:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177278.457218:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.457228:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177278.457231:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.457239:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177278.457240:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177278.457245:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177278.457248:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177278.457251:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302177997
00000100:00000010:2.0:1786177278.457257:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25480.
00000100:00000001:2.0:1786177278.457263:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177278.457268:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302177997
00000100:00000010:2.0:1786177278.457271:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f250c0.
00000100:00000001:2.0:1786177278.457275:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786177278.457278:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177278.457280:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177278.457282:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177278.457287:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177278/real 1786177278]  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177278.457296:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177278.457297:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177278.457299:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177278.457301:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177278.457303:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177278.457305:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177278.457308:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177278.457322:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.457328:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177278.457330:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.457332:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177278.457334:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177278.457338:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.457345:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177278.457350:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177278.457374:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7501
00000400:00000200:0.0:1786177278.457379:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177278.457381:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177278.457384:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177278.457387:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457389:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177278.457391:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457393:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177278.457395:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177278.457397:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177278.457398:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177278.457400:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177278.457402:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177278.457404:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177278.457407:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457409:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177278.457411:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457413:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177278.457415:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177278.457417:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457420:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177278.457421:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177278.457426:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177278.457428:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.457430:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177278.457433:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177278.457436:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.457438:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177278.457441:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419f80.
00000020:00000040:0.0:1786177278.457443:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177278.457446:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177278.457447:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.457449:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177278.457450:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177278.457452:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177278.457456:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4c00.
00000100:00000001:0.0:1786177278.457459:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177278.457464:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177278.457472:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177278.457473:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177278.457477:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ad753ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177278.457501:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177278.457503:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177278.457506:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177278.457508:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177278.457542:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177278.457544:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177278.457547:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7501
00000400:00000200:0.0:1786177278.457550:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177278.457552:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177278.457554:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177278.457557:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457559:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177278.457561:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457563:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177278.457565:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177278.457566:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177278.457568:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177278.457569:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177278.457571:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177278.457573:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177278.457576:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457578:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177278.457580:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457582:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177278.457584:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786177278.457586:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457588:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177278.457590:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177278.457593:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177278.457595:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.457597:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177278.457599:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177278.457601:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.457602:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177278.457606:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418700.
00000020:00000040:0.0:1786177278.457608:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177278.457610:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177278.457611:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.457613:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177278.457615:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177278.457616:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177278.457620:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:0.0:1786177278.457623:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177278.457627:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177278.457633:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177278.457635:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177278.457639:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bf66f89d] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177278.457645:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177278.457648:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177278.457651:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177278.457652:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177278.457654:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177278.457656:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177278.457658:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7501
00000400:00000200:0.0:1786177278.457661:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177278.457664:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177278.457666:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177278.457669:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457671:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177278.457673:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457675:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177278.457677:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177278.457678:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177278.457679:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177278.457681:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177278.457683:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177278.457684:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177278.457687:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457689:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177278.457691:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457692:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177278.457695:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177278.457697:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457699:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177278.457701:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177278.457703:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177278.457704:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.457706:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177278.457708:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177278.457710:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.457712:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177278.457715:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41bb80.
00000020:00000040:0.0:1786177278.457717:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177278.457719:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177278.457720:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.457722:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177278.457723:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177278.457724:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177278.457727:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000100:00000001:0.0:1786177278.457730:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177278.457733:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177278.457740:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177278.457742:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177278.457745:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9983a05] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177278.457752:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177278.457755:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177278.457758:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177278.457759:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177278.457760:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177278.457762:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177278.457764:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7501
00000400:00000200:0.0:1786177278.457767:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177278.457770:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177278.457772:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177278.457774:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457776:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177278.457777:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457779:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177278.457781:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177278.457782:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177278.457784:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177278.457786:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177278.457787:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:0.0:1786177278.457789:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786177278.457791:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457793:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177278.457795:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457797:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177278.457799:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177278.457802:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177278.457804:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177278.457806:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177278.457808:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177278.457809:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.457811:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177278.457812:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177278.457814:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.457815:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177278.457818:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419c00.
00000020:00000040:0.0:1786177278.457820:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177278.457822:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177278.457823:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177278.457824:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177278.457825:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177278.457827:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177278.457829:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000100:00000001:0.0:1786177278.457832:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177278.457835:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177278.457842:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177278.457844:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177278.457847:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cf890c1c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177278.457854:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177278.457858:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7511)
00000100:00000001:2.0:1786177278.458010:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177278.458014:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.458016:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177278.458017:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.458019:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177278.458021:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177278.458024:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177278.458027:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177278.458030:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177278.458035:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177278.458043:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856358912:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177278.458047:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.458049:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177278.458052:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418380 x1872934856358912/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458064:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177278.458067:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177278.458069:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177278.458072:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177278.458074:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177278.458076:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1786177278.458078:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177278.458081:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
02000000:00000001:2.0:1786177278.458085:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177278.458086:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177278.458088:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418380.
00000100:00000001:2.0:1786177278.458091:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177278.458093:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.458095:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.458101:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177278.458103:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177278.458105:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177278.458106:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177278.458108:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177278.458111:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458117:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177278.458119:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177278.458121:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177278.458123:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177278.458125:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177278.458127:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177278.458130:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177278.458132:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177278.458134:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177278.458136:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177278.458139:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177278.458140:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177278.458144:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177278.458147:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177278.458150:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177278.458153:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856359040, portal 10
00000100:00000001:2.0:1786177278.458155:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177278.458156:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761662848 : -114547947888768 : ffff97d1ba419f80)
00000100:00000040:2.0:1786177278.458162:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458168:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177278.458170:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177278.458172:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177278.458175:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856359040, offset 0
00000400:00000200:2.0:1786177278.458194:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177278.458198:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177278.458202:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177278.458205:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177278.458207:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177278.458209:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177278.458211:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177278.458215:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254976:254976:256:4294967295] 192.168.202.110@tcp LPNI seq info [254965:254965:8:4294967295]
00000400:00000200:2.0:1786177278.458219:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177278.458225:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177278.458229:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177278.458233:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177278.458238:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177278.458240:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1786177278.458245:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177278.458248:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1786177278.458251:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177278.458253:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177278.458257:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177278.458260:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177278.458263:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177278.458265:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177278.458268:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177278.458271:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458279:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177278.458281:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458287:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177278.458289:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177278.458292:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177278.458296:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:2.0:1786177278.458299:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177278.458301:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302174218
00000100:00000001:2.0:1786177278.458304:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302177997
00000100:00000001:2.0:1786177278.458306:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302174218
00000100:00000001:2.0:1786177278.458308:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302177997
00000100:00000001:2.0:1786177278.458310:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177278.458312:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177278.458314:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177278.458317:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177278/real 1786177278]  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177278.458324:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177278.458326:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177278.458327:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177278.458330:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177278.458331:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177278.458334:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177278.458337:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177278.458345:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458596:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177278.458598:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.458600:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177278.458601:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177278.458606:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458613:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177278.458616:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458623:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177278.458625:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.458627:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177278.458628:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.458630:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177278.458632:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177278.458634:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177278.458637:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177278.458639:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177278.458643:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177278.458651:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359040:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177278.458655:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177278.458658:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458664:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177278.458665:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177278.458667:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177278.458669:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177278.458671:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177278.458674:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177278.458677:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177278.458678:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177278.458680:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177278.458681:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177278.458685:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177278.458687:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177278.458690:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177278.458693:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177278.458696:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177278.458699:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856359168, portal 10
00000100:00000001:2.0:1786177278.458701:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177278.458702:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761656576 : -114547947895040 : ffff97d1ba418700)
00000100:00000040:2.0:1786177278.458707:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458713:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177278.458715:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177278.458717:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177278.458719:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856359168, offset 0
00000400:00000200:2.0:1786177278.458723:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177278.458726:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177278.458730:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177278.458733:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177278.458735:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177278.458737:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177278.458739:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177278.458743:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254977:254977:256:4294967295] 192.168.202.110@tcp LPNI seq info [254966:254966:8:4294967295]
00000400:00000200:2.0:1786177278.458747:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177278.458753:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177278.458758:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177278.458760:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177278.458764:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177278.458767:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1786177278.458772:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177278.458774:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1786177278.458777:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177278.458779:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177278.458782:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177278.458786:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177278.458788:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177278.458790:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177278.458792:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177278.458795:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458802:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177278.458805:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458811:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177278.458812:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177278.458815:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177278.458818:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:2.0:1786177278.458821:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177278.458823:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302174344
00000100:00000001:2.0:1786177278.458825:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302177997
00000100:00000001:2.0:1786177278.458827:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302174344
00000100:00000001:2.0:1786177278.458829:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302177997
00000100:00000001:2.0:1786177278.458831:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177278.458833:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177278.458834:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177278.458837:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177278/real 1786177278]  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177278.458845:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177278.458847:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177278.458848:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177278.458850:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177278.458852:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177278.458853:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177278.458857:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177278.458865:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458871:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177278.458872:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.458874:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177278.458875:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177278.458879:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458885:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177278.458889:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458895:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177278.458898:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.458899:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177278.458901:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.458902:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177278.458904:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177278.458907:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177278.458909:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177278.458912:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177278.458916:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177278.458923:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359168:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177278.458927:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177278.458930:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458936:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177278.458937:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177278.458939:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177278.458940:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177278.458942:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177278.458945:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177278.458948:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177278.458950:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177278.458951:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177278.458953:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177278.458955:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab0000.
02000000:00000001:2.0:1786177278.458957:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177278.458960:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177278.458963:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177278.458965:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177278.458967:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856359296, portal 4
00000100:00000001:2.0:1786177278.458969:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177278.458970:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761670016 : -114547947881600 : ffff97d1ba41bb80)
00000100:00000040:2.0:1786177278.458975:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.458993:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177278.458996:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177278.458998:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177278.459000:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856359296, offset 0
00000400:00000200:2.0:1786177278.459004:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177278.459008:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177278.459011:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177278.459013:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177278.459015:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177278.459018:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177278.459020:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177278.459023:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254978:254978:256:4294967295] 192.168.202.110@tcp LPNI seq info [254967:254967:8:4294967295]
00000400:00000200:2.0:1786177278.459027:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177278.459032:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177278.459036:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177278.459039:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177278.459043:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177278.459046:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1786177278.459050:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177278.459052:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1786177278.459056:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177278.459057:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177278.459060:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177278.459063:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177278.459066:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177278.459068:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177278.459070:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177278.459073:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459080:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177278.459083:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459089:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177278.459090:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177278.459093:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177278.459096:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:2.0:1786177278.459099:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177278.459100:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302174219
00000100:00000001:2.0:1786177278.459103:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302177997
00000100:00000001:2.0:1786177278.459104:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302174219
00000100:00000001:2.0:1786177278.459106:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302177997
00000100:00000001:2.0:1786177278.459108:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177278.459110:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177278.459112:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177278.459115:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177278/real 1786177278]  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177278.459122:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177278.459124:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177278.459125:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177278.459127:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177278.459129:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177278.459130:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177278.459133:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177278.459141:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459146:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177278.459148:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.459149:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177278.459150:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177278.459154:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459160:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177278.459163:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459170:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177278.459172:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.459174:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177278.459175:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.459176:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177278.459178:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177278.459180:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177278.459183:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177278.459185:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177278.459188:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177278.459195:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359296:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177278.459198:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177278.459201:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459208:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177278.459209:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177278.459211:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177278.459213:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177278.459215:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177278.459217:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177278.459220:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177278.459221:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177278.459223:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177278.459224:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177278.459227:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1c00.
02000000:00000001:2.0:1786177278.459229:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177278.459232:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177278.459234:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177278.459237:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177278.459239:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856359424, portal 4
00000100:00000001:2.0:1786177278.459241:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177278.459242:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761661952 : -114547947889664 : ffff97d1ba419c00)
00000100:00000040:2.0:1786177278.459247:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459253:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177278.459254:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177278.459256:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177278.459258:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856359424, offset 0
00000400:00000200:2.0:1786177278.459262:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177278.459265:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177278.459268:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177278.459271:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177278.459273:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177278.459275:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177278.459277:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177278.459279:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254979:254979:256:4294967295] 192.168.202.110@tcp LPNI seq info [254968:254968:8:4294967295]
00000400:00000200:2.0:1786177278.459283:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177278.459288:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177278.459293:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177278.459296:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177278.459299:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177278.459301:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1786177278.459305:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177278.459308:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1786177278.459311:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177278.459313:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177278.459316:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177278.459319:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177278.459322:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177278.459323:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177278.459326:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177278.459329:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459342:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177278.459345:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459352:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177278.459353:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177278.459355:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177278.459358:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:2.0:1786177278.459361:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177278.459362:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302174227
00000100:00000001:2.0:1786177278.459365:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302177997
00000100:00000001:2.0:1786177278.459367:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302174227
00000100:00000001:2.0:1786177278.459369:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302177997
00000100:00000001:2.0:1786177278.459371:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177278.459372:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177278.459374:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177278.459377:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177278/real 1786177278]  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177278.459383:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177278.459385:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177278.459386:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177278.459388:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177278.459390:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177278.459392:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177278.459395:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177278.459403:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459409:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177278.459411:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.459412:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177278.459413:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177278.459417:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459424:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177278.459427:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459434:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177278.459436:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.459437:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177278.459438:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.459440:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177278.459441:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177278.459443:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177278.459446:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177278.459448:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177278.459452:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177278.459458:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359424:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177278.459462:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.459464:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177278.459467:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419f80 x1872934856359040/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459473:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177278.459475:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177278.459477:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177278.459479:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177278.459481:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177278.459483:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177278.459485:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177278.459486:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:2.0:1786177278.459488:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177278.459489:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177278.459491:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419f80.
00000100:00000001:2.0:1786177278.459494:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177278.459495:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.459497:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177278.459500:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418700 x1872934856359168/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459506:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177278.459507:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177278.459526:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177278.459528:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177278.459529:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177278.459530:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177278.459532:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177278.459533:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:2.0:1786177278.459536:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177278.459537:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177278.459538:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418700.
00000100:00000001:2.0:1786177278.459540:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177278.459542:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.459543:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177278.459546:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41bb80 x1872934856359296/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459552:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177278.459554:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177278.459555:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab0000.
02000000:00000001:2.0:1786177278.459558:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177278.459559:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177278.459560:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177278.459561:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177278.459562:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:2.0:1786177278.459564:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177278.459566:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177278.459567:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41bb80.
00000100:00000001:2.0:1786177278.459569:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177278.459570:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.459571:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177278.459574:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419c00 x1872934856359424/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177278.459580:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177278.459582:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177278.459583:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:2.0:1786177278.459584:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177278.459585:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177278.459586:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177278.459588:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177278.459589:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:2.0:1786177278.459590:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177278.459591:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177278.459593:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419c00.
00000100:00000001:2.0:1786177278.459595:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177278.459596:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.459598:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177278.459602:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177278.459603:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177278.459605:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177278.459607:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177279.032954:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177279.032963:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177279.032980:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177279.032982:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177279.032986:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177279.032987:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177279.033205:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177279.033210:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177279.033214:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177279.480673:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177279.480684:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177279.480686:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177279.480695:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177279.480700:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177279.480704:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177279.480708:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177279.480712:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177279.480715:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177280.056728:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177280.056745:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177280.056753:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177280.056754:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177280.056758:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177280.056759:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177280.056999:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177280.057003:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177280.057006:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177280.504697:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177280.504708:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177280.504710:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177280.504716:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177280.504721:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177280.504726:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177280.504729:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177280.504742:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177280.504746:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177281.080688:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177281.080700:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177281.080703:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177281.080707:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177281.080714:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177281.080717:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177281.080719:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177281.080722:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177281.080791:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177281.080795:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177281.080797:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177281.080799:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177281.080801:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177281.080803:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177281.080805:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177281.080807:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177281.080812:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177281.080814:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177281.080916:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177281.080923:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177281.080928:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1786177281.528665:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177281.528677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177281.528680:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177281.528689:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177281.528694:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177281.528698:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177281.528701:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177281.528703:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177281.528706:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177282.104678:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177282.104684:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177282.104690:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177282.104691:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177282.104694:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177282.104695:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177282.104824:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177282.104828:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177282.104831:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177282.296674:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302177997, at 4302178048
00000100:00000010:0.0:1786177282.296692:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25480.
00000100:00000040:1.0:1786177282.297256:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302177997, at 4302178048
00000100:00000010:1.0:1786177282.297263:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f250c0.
00000100:00000001:2.0:1786177282.556219:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1786177282.556227:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177282.556230:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177282.556236:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177282.556239:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177282.556244:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177282.556246:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177282.556249:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177282.556252:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177283.128636:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177283.128642:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177283.128648:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177283.128650:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177283.128653:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177283.128654:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177283.128748:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177283.128752:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177283.128755:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786177283.576656:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177283.576665:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177283.576672:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:2.0:1786177283.576673:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177283.576678:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.576678:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:0.0:1786177283.576685:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786177283.576686:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177283.576693:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177283.576694:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786177283.576699:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177283.576705:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1786177283.576705:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7506
00000100:00000001:0.0:1786177283.576710:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177283.576714:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177283.576718:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177283.576727:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177283.576730:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177283.576733:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.576736:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177283.576738:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.576740:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177283.576743:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177283.576745:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177283.576748:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177283.576751:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177283.576753:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177283.576755:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177283.576757:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.576758:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177283.576760:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.576770:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177283.576773:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177283.576779:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177283.576782:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177283.576786:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177283.576790:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177283.576794:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.576796:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177283.576798:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177283.576809:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786177283.576814:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177283.576818:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177283.576820:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.576832:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177283.576836:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177283.576841:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177283.576950:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786177283.576959:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177283.576971:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177283.576982:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177283.576988:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177283.576993:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177283.577024:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177283.577027:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177283.577030:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177283.577032:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177283.577033:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177283.577036:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177283.577039:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7506
00000400:00000200:2.0:1786177283.577043:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177283.577046:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177283.577049:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177283.577052:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577054:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177283.577055:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577057:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177283.577059:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177283.577061:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177283.577063:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177283.577065:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177283.577068:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177283.577069:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177283.577072:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577074:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177283.577076:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577078:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177283.577081:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177283.577083:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577086:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177283.577088:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177283.577094:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177283.577096:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.577098:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177283.577103:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177283.577107:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.577109:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177283.577113:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786177283.577116:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177283.577118:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177283.577120:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.577121:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177283.577123:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177283.577125:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177283.577129:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786177283.577132:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177283.577138:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177283.577145:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177283.577147:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177283.577150:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177283.577164:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177283.577166:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177283.577169:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177283.577171:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177283.577172:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177283.577175:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177283.577178:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7506
00000400:00000200:2.0:1786177283.577181:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177283.577184:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177283.577187:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177283.577189:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577191:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177283.577192:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577193:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177283.577195:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177283.577196:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177283.577197:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177283.577198:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177283.577199:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786177283.577201:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786177283.577202:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577203:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177283.577204:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577206:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177283.577207:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786177283.577209:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577210:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177283.577212:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177283.577214:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177283.577215:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.577217:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177283.577218:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177283.577220:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.577221:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177283.577224:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786177283.577225:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177283.577227:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177283.577227:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.577229:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177283.577230:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177283.577231:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177283.577235:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786177283.577238:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177283.577243:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177283.577250:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177283.577252:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177283.577255:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177283.577262:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177283.577265:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177283.577268:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177283.577270:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177283.577272:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177283.577275:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177283.577277:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7506
00000400:00000200:2.0:1786177283.577281:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177283.577284:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177283.577286:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177283.577288:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577289:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177283.577291:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577292:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177283.577293:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177283.577294:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177283.577295:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177283.577296:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177283.577297:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177283.577299:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786177283.577301:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577302:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177283.577303:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.577305:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177283.577306:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:3.0:1786177283.577403:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177283.577407:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177283.577410:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177283.577417:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.577428:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177283.577430:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177283.577434:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177283.577437:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177283.577440:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177283.577443:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856359552:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177283.577448:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177283.577453:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177283.577457:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177283.577460:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177283.577466:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177283.577469:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177283.577476:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177283.577481:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177283.577489:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177283.577500:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856359552, portal 25
00000100:00000001:3.0:1786177283.577502:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177283.577504:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:3.0:1786177283.577540:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.577548:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177283.577551:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177283.577555:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177283.577560:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856359552, offset 0
00000400:00000200:3.0:1786177283.577568:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177283.577582:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177283.577590:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177283.577597:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177283.577602:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177283.577607:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177283.577611:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177283.577615:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254980:254980:256:4294967295] 192.168.202.110@tcp LPNI seq info [254969:254969:8:4294967295]
00000400:00000200:3.0:1786177283.577620:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177283.577632:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177283.577639:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177283.577646:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177283.577654:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177283.577658:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786177283.577665:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177283.577669:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1786177283.577674:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177283.577677:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177283.577685:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177283.577690:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177283.577697:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177283.577700:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177283.577706:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177283.577710:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.577720:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177283.577724:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.577732:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177283.577734:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177283.577744:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177283.577748:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177283.577752:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302183118
00000100:00000010:3.0:1786177283.577758:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37000.
00000100:00000001:3.0:1786177283.577768:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177283.577775:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302183118
00000100:00000010:3.0:1786177283.577778:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37240.
00000100:00000001:3.0:1786177283.577781:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177283.577785:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177283.577787:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177283.577790:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177283.577795:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177283/real 1786177283]  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177283.577803:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177283.577805:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177283.577807:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177283.577811:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177283.577812:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177283.577815:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177283.577818:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177283.577833:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.577848:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177283.577850:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.577853:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177283.577856:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177283.577861:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.577868:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177283.577873:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.577881:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177283.577883:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.577886:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177283.577887:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.577890:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177283.577893:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177283.577895:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177283.577899:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177283.577902:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177283.577907:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177283.577916:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856359552:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177283.577922:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177283.577925:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.577939:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177283.577940:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177283.577942:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177283.577944:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177283.577945:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177283.577947:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359680:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177283.577949:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177283.577951:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177283.577952:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177283.577953:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177283.577956:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177283.577957:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177283.577960:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177283.577962:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177283.577965:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177283.577967:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856359680, portal 10
00000100:00000001:3.0:1786177283.577968:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177283.577970:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:3.0:1786177283.577976:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.577982:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177283.577985:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177283.577987:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177283.577989:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856359680, offset 0
00000400:00000200:3.0:1786177283.577993:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177283.577997:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177283.578001:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177283.578004:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177283.578006:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177283.578009:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177283.578011:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177283.578014:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254981:254981:256:4294967295] 192.168.202.110@tcp LPNI seq info [254970:254970:8:4294967295]
00000400:00000200:3.0:1786177283.578018:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177283.578024:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177283.578029:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177283.578033:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177283.578037:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177283.578040:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786177283.578044:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177283.578046:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1786177283.578050:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177283.578052:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177283.578055:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177283.578058:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177283.578061:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177283.578063:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177283.578066:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177283.578070:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578083:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177283.578086:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578091:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177283.578092:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177283.578095:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177283.578099:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177283.578102:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177283.578104:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302179338
00000100:00000001:3.0:1786177283.578107:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302183118
00000100:00000001:3.0:1786177283.578109:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302179338
00000100:00000001:3.0:1786177283.578111:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302183118
00000100:00000001:3.0:1786177283.578113:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177283.578115:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177283.578116:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177283.578120:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177283/real 1786177283]  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177283.578127:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177283.578128:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177283.578130:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177283.578133:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177283.578134:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177283.578137:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177283.578140:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177283.578147:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578152:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177283.578154:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.578156:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177283.578158:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177283.578161:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578167:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177283.578171:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578178:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177283.578181:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.578182:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177283.578184:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.578185:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177283.578187:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177283.578189:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177283.578193:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177283.578195:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177283.578199:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177283.578207:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359680:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177283.578211:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177283.578214:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578221:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177283.578223:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177283.578224:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177283.578225:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177283.578227:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177283.578229:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177283.578233:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177283.578234:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177283.578236:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177283.578238:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177283.578243:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177283.578245:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177283.578248:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177283.578251:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177283.578254:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177283.578257:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856359808, portal 10
00000100:00000001:3.0:1786177283.578259:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177283.578260:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:3.0:1786177283.578266:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578273:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177283.578275:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177283.578277:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177283.578279:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856359808, offset 0
00000400:00000200:3.0:1786177283.578283:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177283.578286:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177283.578290:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177283.578293:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177283.578295:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177283.578297:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177283.578299:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177283.578302:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254982:254982:256:4294967295] 192.168.202.110@tcp LPNI seq info [254971:254971:8:4294967295]
00000400:00000200:3.0:1786177283.578306:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177283.578312:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177283.578316:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177283.578319:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177283.578323:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177283.578326:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786177283.578330:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177283.578332:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1786177283.578336:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177283.578338:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177283.578341:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177283.578344:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177283.578347:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177283.578349:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177283.578351:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177283.578354:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578362:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177283.578365:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578371:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177283.578373:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177283.578376:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177283.578379:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177283.578382:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177283.578385:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302179463
00000100:00000001:3.0:1786177283.578386:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302183118
00000100:00000001:3.0:1786177283.578388:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302179463
00000100:00000001:3.0:1786177283.578390:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302183118
00000100:00000001:3.0:1786177283.578391:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177283.578392:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177283.578393:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177283.578396:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177283/real 1786177283]  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177283.578403:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177283.578405:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177283.578406:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177283.578408:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177283.578410:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177283.578412:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177283.578415:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177283.578423:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578430:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177283.578431:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.578433:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177283.578435:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177283.578438:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578444:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177283.578447:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578451:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177283.578453:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.578454:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177283.578455:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.578456:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177283.578457:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177283.578459:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177283.578460:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177283.578461:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177283.578464:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177283.578472:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359808:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177283.578475:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.578477:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177283.578479:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856359552/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.578482:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177283.578485:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177283.578487:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177283.578489:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177283.578490:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177283.578492:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177283.578494:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177283.578496:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:3.0:1786177283.578500:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177283.578501:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177283.578502:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:3.0:1786177283.578505:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177283.578506:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.578507:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00080000:2.0:1786177283.578762:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177283.578765:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177283.578766:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177283.578769:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177283.578771:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.578773:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177283.578776:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177283.578777:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.578779:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177283.578783:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786177283.578785:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177283.578788:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177283.578789:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.578790:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177283.578792:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177283.578794:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177283.578797:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786177283.578801:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177283.578805:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177283.578811:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177283.578813:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177283.578816:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177283.578821:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177283.578832:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177283.578839:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177283.578840:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177283.578842:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177283.578845:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177283.578848:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7506
00000400:00000200:2.0:1786177283.578853:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177283.578857:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177283.578859:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177283.578862:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.578864:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177283.578867:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.578869:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177283.578872:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177283.578873:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177283.578876:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177283.578879:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786177283.578880:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786177283.578882:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177283.578884:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.578885:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177283.578887:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177283.578888:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177283.578890:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786177283.578892:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177283.578893:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177283.578894:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177283.578896:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177283.578897:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.578898:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177283.578899:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177283.578901:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.578902:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177283.578904:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786177283.578906:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177283.578907:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177283.578909:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177283.578910:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177283.578911:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177283.578912:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177283.578915:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786177283.578917:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177283.578920:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177283.578924:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177283.578925:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177283.578950:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177283.578956:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177283.578959:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7517)
00000100:00000040:3.0:1786177283.579278:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856359680/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.579287:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177283.579289:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177283.579291:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177283.579295:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177283.579297:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177283.579299:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177283.579301:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177283.579303:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:3.0:1786177283.579306:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177283.579307:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177283.579308:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:3.0:1786177283.579313:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177283.579314:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.579316:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177283.579319:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856359808/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.579325:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177283.579326:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177283.579328:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177283.579330:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177283.579331:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177283.579332:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177283.579334:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177283.579336:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:3.0:1786177283.579338:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177283.579339:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177283.579340:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:3.0:1786177283.579342:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177283.579343:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.579346:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.579354:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177283.579356:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177283.579359:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177283.579361:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177283.579363:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177283.579366:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.579373:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177283.579374:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177283.579377:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177283.579379:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177283.579381:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177283.579442:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177283.579446:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177283.579448:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177283.579450:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177283.579453:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177283.579457:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177283.579459:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177283.579464:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177283.579467:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177283.579471:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177283.579475:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856359936, portal 4
00000100:00000001:3.0:1786177283.579478:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177283.579479:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000040:3.0:1786177283.579485:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.579492:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177283.579494:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177283.579497:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177283.579500:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856359936, offset 0
00000400:00000200:3.0:1786177283.579504:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177283.579526:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177283.579532:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177283.579535:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177283.579537:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177283.579540:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177283.579542:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177283.579545:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254983:254983:256:4294967295] 192.168.202.110@tcp LPNI seq info [254972:254972:8:4294967295]
00000400:00000200:3.0:1786177283.579549:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177283.579557:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177283.579561:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177283.579565:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177283.579569:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177283.579572:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786177283.579576:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177283.579578:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1786177283.579582:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177283.579583:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177283.579587:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177283.579591:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177283.579595:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177283.579597:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177283.579599:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177283.579603:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.579611:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177283.579615:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.579621:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177283.579622:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177283.579627:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177283.579631:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177283.579634:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177283.579636:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302179340
00000100:00000001:3.0:1786177283.579639:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302183118
00000100:00000001:3.0:1786177283.579641:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302179340
00000100:00000001:3.0:1786177283.579643:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302183118
00000100:00000001:3.0:1786177283.579645:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177283.579647:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177283.579649:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177283.579653:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177283/real 1786177283]  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177283.579662:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177283.579663:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177283.579665:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177283.579667:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177283.579669:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177283.579671:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177283.579675:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177283.579683:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.579689:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177283.579691:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.579693:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177283.579695:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177283.579698:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.579705:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177283.579709:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.579715:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177283.579717:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.579719:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177283.579720:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.579722:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177283.579724:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177283.579726:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177283.579729:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177283.579731:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177283.579735:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177283.579743:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856359936:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177283.579747:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177283.579750:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.579761:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177283.579763:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177283.579765:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177283.579767:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177283.579769:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177283.579771:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856360064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177283.579775:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177283.579777:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177283.579778:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177283.579780:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177283.579783:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177283.579785:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177283.579788:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177283.579791:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177283.579793:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177283.579796:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856360064, portal 4
00000100:00000001:3.0:1786177283.579797:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177283.579798:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:3.0:1786177283.579802:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.579806:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177283.579808:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177283.579809:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177283.579810:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856360064, offset 0
00000400:00000200:3.0:1786177283.579813:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177283.579815:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177283.579817:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177283.579819:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177283.579821:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177283.579822:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177283.581461:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177283.581468:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254984:254984:256:4294967295] 192.168.202.110@tcp LPNI seq info [254973:254973:8:4294967295]
00000400:00000200:3.0:1786177283.581473:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177283.581483:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177283.581489:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177283.581495:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177283.581501:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177283.581503:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1786177283.581522:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177283.581525:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1786177283.581529:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177283.581531:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177283.581536:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177283.581540:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177283.581545:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177283.581547:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177283.581550:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177283.581555:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.581565:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177283.581568:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.581575:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177283.581576:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177283.581582:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177283.581586:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177283.581589:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177283.581592:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302179350
00000100:00000001:3.0:1786177283.581595:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302183118
00000100:00000001:3.0:1786177283.581597:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302179350
00000100:00000001:3.0:1786177283.581599:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302183118
00000100:00000001:3.0:1786177283.581602:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177283.581605:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177283.581607:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177283.581612:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177283/real 1786177283]  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177283.581671:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177283.581672:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177283.581675:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177283.581678:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177283.581680:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177283.581683:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177283.581686:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177283.581694:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.581701:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177283.581703:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.581706:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177283.581707:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177283.581711:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.581718:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177283.581722:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.581728:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177283.581731:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.581732:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177283.581733:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.581734:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177283.581736:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177283.581738:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177283.581741:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177283.581743:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177283.581745:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177283.581750:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856360064:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177283.581752:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.581754:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177283.581756:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856359936/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.581761:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177283.581762:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177283.581765:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177283.581767:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177283.581769:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177283.581770:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177283.581772:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177283.581774:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:3.0:1786177283.581776:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177283.581778:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177283.581779:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:3.0:1786177283.581781:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177283.581782:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.581783:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177283.581785:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856360064/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177283.581790:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177283.581791:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177283.581792:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177283.581794:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177283.581795:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177283.581796:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177283.581797:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177283.581798:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:3.0:1786177283.581799:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177283.581800:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177283.581801:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:3.0:1786177283.581803:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177283.581803:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.581805:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177283.581810:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177283.581811:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177283.581812:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177283.581814:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177284.152645:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177284.152651:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177284.152656:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177284.152657:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177284.152661:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177284.152664:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177284.152667:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177284.152670:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177284.152673:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177284.152676:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177284.152678:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177284.152680:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177284.154407:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177284.154416:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177284.154422:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177284.600635:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177284.600647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177284.600649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177284.600657:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177284.600662:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177284.600666:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177284.600669:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177284.600673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177284.600676:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177285.176686:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177285.176695:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177285.176701:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177285.176703:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177285.176706:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177285.176707:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177285.176787:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177285.176792:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177285.176796:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177285.624715:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177285.624726:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177285.624729:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177285.624736:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177285.624740:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177285.624746:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177285.624749:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177285.624753:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177285.624756:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177286.200640:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177286.200645:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177286.200649:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177286.200650:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177286.200652:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177286.200653:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177286.200709:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177286.200711:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177286.200713:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177286.648768:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177286.648777:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177286.648779:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177286.648784:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177286.648787:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177286.648792:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177286.648796:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177286.648800:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177286.648803:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177287.224638:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177287.224646:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177287.224652:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177287.224653:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177287.224657:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177287.224658:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177287.224757:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177287.224763:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177287.224769:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177287.416623:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302183118, at 4302183168
00000100:00000010:1.0:1786177287.416635:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37240.
00000100:00000040:0.0:1786177287.416685:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302183118, at 4302183168
00000100:00000010:0.0:1786177287.416695:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37000.
00000100:00000001:0.0:1786177287.673130:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177287.673138:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177287.673140:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177287.673144:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177287.673147:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177287.673150:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177287.673152:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177287.673154:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177287.673156:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177288.248649:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177288.248657:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177288.248662:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177288.248663:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177288.248667:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177288.248668:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177288.248727:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177288.248731:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177288.248734:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177288.696600:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177288.696609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177288.696611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177288.696611:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:0.0:1786177288.696617:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786177288.696619:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177288.696620:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177288.696622:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177288.696624:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177288.696627:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1786177288.696628:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177288.696629:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177288.696633:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.696635:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177288.696645:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7512
00000400:00000200:2.0:1786177288.696655:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177288.696662:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177288.696666:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177288.696669:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.696671:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177288.696673:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.696675:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177288.696679:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177288.696681:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177288.696684:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177288.696687:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177288.696689:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177288.696692:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177288.696694:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.696695:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177288.696697:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.696698:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177288.696701:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177288.696703:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177288.696706:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177288.696709:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177288.696712:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177288.696715:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.696717:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177288.696718:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177288.696726:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786177288.696730:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177288.696733:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177288.696735:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.696738:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177288.696741:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177288.696745:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177288.696752:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000001:2.0:1786177288.696758:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177288.696766:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177288.696775:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177288.696790:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177288.696796:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177288.696820:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177288.696824:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177288.696827:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177288.696837:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177288.696840:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177288.696843:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177288.696846:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7512
00000400:00000200:2.0:1786177288.696850:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177288.696853:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177288.696855:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177288.696858:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.696860:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177288.696861:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.696862:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177288.696864:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177288.696865:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177288.696866:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177288.696868:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177288.696870:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177288.696872:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177288.696875:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.696877:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177288.696888:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.696889:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177288.696892:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177288.696894:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177288.696896:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177288.696898:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177288.696904:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177288.696906:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.696908:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177288.696912:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177288.696915:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.696916:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177288.696917:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.696923:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000010:2.0:1786177288.696924:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786177288.696928:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:3.0:1786177288.696928:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:2.0:1786177288.696933:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177288.696935:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.696938:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:3.0:1786177288.696939:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177288.696940:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177288.696943:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177288.696950:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786177288.696955:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177288.696961:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177288.696965:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177288.696968:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.696972:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1786177288.696975:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1786177288.696976:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177288.696979:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1786177288.696983:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177288.696985:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177288.696994:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856360192:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177288.697000:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.697002:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:2.0:1786177288.697004:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:3.0:1786177288.697008:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:2.0:1786177288.697009:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177288.697013:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:3.0:1786177288.697013:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177288.697016:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:3.0:1786177288.697018:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786177288.697020:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177288.697025:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7512
02000000:00000010:3.0:1786177288.697026:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
00000400:00000200:2.0:1786177288.697031:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:3.0:1786177288.697031:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177288.697037:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177288.697040:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1786177288.697042:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:2.0:1786177288.697045:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.697049:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1786177288.697049:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000040:2.0:1786177288.697053:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.697056:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177288.697059:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177288.697060:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000400:00000010:3.0:1786177288.697060:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000020:00000001:2.0:1786177288.697064:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177288.697067:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177288.697069:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000200:3.0:1786177288.697070:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856360192, portal 25
00000020:00000001:2.0:1786177288.697072:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:3.0:1786177288.697075:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000040:2.0:1786177288.697078:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177288.697079:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454181632 : -114548255369984 : ffff97d1a7edd500)
00000100:00000001:2.0:1786177288.697082:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177288.697086:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.697091:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:3.0:1786177288.697091:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177288.697096:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786177288.697103:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177288.697108:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.697109:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:2.0:1786177288.697111:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177288.697116:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1786177288.697116:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177288.697121:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177288.697125:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856360192, offset 0
02000000:00000001:2.0:1786177288.697131:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177288.697135:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177288.697136:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177288.697139:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177288.697143:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.697145:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177288.697147:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:2.0:1786177288.697150:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000020:00000040:2.0:1786177288.697154:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:3.0:1786177288.697156:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:2.0:1786177288.697158:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177288.697161:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177288.697163:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786177288.697164:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177288.697167:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1786177288.697170:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177288.697175:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177288.697178:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:2.0:1786177288.697179:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177288.697183:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254985:254985:256:4294967295] 192.168.202.110@tcp LPNI seq info [254974:254974:8:4294967295]
02000000:00000010:2.0:1786177288.697188:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000400:00000200:3.0:1786177288.697192:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177288.697195:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177288.697202:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177288.697211:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786177288.697217:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177288.697221:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177288.697225:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177288.697229:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:3.0:1786177288.697233:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000100:00000001:2.0:1786177288.697242:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177288.697245:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:2.0:1786177288.697247:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:3.0:1786177288.697253:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000100:00080000:2.0:1786177288.697254:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177288.697256:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177288.697260:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:3.1:1786177288.697265:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177288.697266:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177288.697272:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7512
00000800:00000200:3.1:1786177288.697273:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000400:00000200:2.0:1786177288.697279:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000100:3.0:1786177288.697284:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177288.697285:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000800:00000010:3.0:1786177288.697288:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000100:00080000:2.0:1786177288.697289:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177288.697294:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177288.697296:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1786177288.697297:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177288.697301:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.697304:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177288.697305:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:2.0:1786177288.697309:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177288.697311:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000400:00000200:3.0:1786177288.697313:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000020:00000001:2.0:1786177288.697314:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000400:00000010:3.0:1786177288.697318:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000020:00000040:2.0:1786177288.697319:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177288.697324:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177288.697327:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:3.0:1786177288.697327:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1786177288.697335:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:3.0:1786177288.697335:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177288.697339:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177288.697343:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.697348:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177288.697353:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177288.697357:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177288.697358:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786177288.697361:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177288.697365:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1786177288.697367:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177288.697369:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177288.697374:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.697378:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177288.697381:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177288.697391:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.697394:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000008:00000001:2.0:1786177288.697398:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.697400:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.697402:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177288.697408:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000400:00000200:3.0:1786177288.697409:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:2.0:1786177288.697412:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177288.697414:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302188237
02000000:00000001:2.0:1786177288.697417:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177288.697419:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.697421:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:3.0:1786177288.697422:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37cc0.
00000100:00000001:3.0:1786177288.697430:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177288.697435:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302188237
00000100:00000010:3.0:1786177288.697437:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37f00.
00000100:00000001:3.0:1786177288.697440:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177288.697443:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177288.697445:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.697447:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177288.697451:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177288/real 1786177288]  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177288.697459:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177288.697460:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177288.697462:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177288.697465:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177288.697466:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177288.697468:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177288.697472:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177288.697479:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697485:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177288.697487:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.697488:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177288.697490:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177288.697494:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697500:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177288.697504:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697533:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177288.697537:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.697538:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177288.697539:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.697541:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177288.697543:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177288.697546:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177288.697550:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177288.697552:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177288.697557:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177288.697564:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856360192:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177288.697568:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.697570:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177288.697573:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd500 x1872934856360192/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697579:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177288.697581:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177288.697584:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177288.697587:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177288.697588:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177288.697590:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177288.697591:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177288.697593:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3400.
02000000:00000001:3.0:1786177288.697597:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177288.697599:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177288.697601:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd500.
00000100:00000001:3.0:1786177288.697604:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177288.697606:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.697608:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.697614:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177288.697616:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.697618:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177288.697619:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177288.697621:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177288.697624:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697631:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177288.697632:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177288.697634:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177288.697636:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177288.697638:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177288.697641:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856360320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177288.697644:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177288.697646:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177288.697648:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177288.697649:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177288.697652:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177288.697654:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177288.697657:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177288.697660:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177288.697663:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177288.697666:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856360320, portal 10
00000100:00000001:3.0:1786177288.697668:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177288.697669:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187904 : -114548255363712 : ffff97d1a7eded80)
00000100:00000040:3.0:1786177288.697676:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697682:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177288.697684:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177288.697686:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177288.697688:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856360320, offset 0
00000400:00000200:3.0:1786177288.697692:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177288.697696:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177288.697699:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177288.697702:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177288.697704:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177288.697706:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177288.697708:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177288.697711:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254986:254986:256:4294967295] 192.168.202.110@tcp LPNI seq info [254975:254975:8:4294967295]
00000400:00000200:3.0:1786177288.697715:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177288.697721:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177288.697725:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177288.697728:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177288.697732:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177288.697734:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786177288.697737:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177288.697739:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1786177288.697742:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177288.697744:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177288.697747:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177288.697750:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177288.697752:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177288.697754:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177288.697756:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177288.697759:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697766:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177288.697769:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697775:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.697776:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177288.697780:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177288.697783:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177288.697786:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177288.697788:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302184458
00000100:00000001:3.0:1786177288.697790:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302188237
00000100:00000001:3.0:1786177288.697792:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302184458
00000100:00000001:3.0:1786177288.697794:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302188237
00000100:00000001:3.0:1786177288.697796:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177288.697797:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.697799:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177288.697802:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177288/real 1786177288]  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177288.697809:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177288.697810:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177288.697812:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177288.697815:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177288.697816:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177288.697818:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177288.697821:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177288.697828:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697833:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177288.697835:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.697836:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177288.697837:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177288.697841:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697846:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177288.697850:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697856:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177288.697858:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.697859:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177288.697860:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.697862:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177288.697864:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177288.697866:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177288.697869:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177288.697871:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177288.697874:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177288.697892:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856360320:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177288.697896:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177288.697899:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697904:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177288.697906:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177288.697907:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177288.697909:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177288.697911:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177288.697914:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856360448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177288.697917:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177288.697918:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177288.697920:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177288.697921:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177288.697925:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177288.697926:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177288.697929:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177288.697931:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177288.697934:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177288.697937:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856360448, portal 10
00000100:00000001:3.0:1786177288.697938:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177288.697939:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454180736 : -114548255370880 : ffff97d1a7edd180)
00000100:00000040:3.0:1786177288.697945:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.697951:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177288.697953:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177288.697955:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177288.697957:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856360448, offset 0
00000400:00000200:3.0:1786177288.697961:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177288.697964:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177288.697967:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177288.697970:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177288.697972:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177288.697974:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177288.697976:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177288.697978:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254987:254987:256:4294967295] 192.168.202.110@tcp LPNI seq info [254976:254976:8:4294967295]
00000400:00000200:3.0:1786177288.697981:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177288.697986:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177288.697990:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177288.697993:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177288.697996:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177288.697998:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786177288.698002:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177288.698004:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1786177288.698007:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177288.698009:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177288.698012:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177288.698015:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177288.698018:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177288.698020:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177288.698022:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177288.698026:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.698032:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177288.698035:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.698039:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.698041:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177288.698043:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177288.698046:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177288.698049:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177288.698050:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302184583
00000100:00000001:3.0:1786177288.698053:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302188237
00000100:00000001:3.0:1786177288.698055:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302184583
00000100:00000001:3.0:1786177288.698057:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302188237
00000100:00000001:3.0:1786177288.698058:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177288.698060:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.698062:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177288.698065:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177288/real 1786177288]  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177288.698072:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177288.698073:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177288.698074:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177288.698076:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177288.698077:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177288.698080:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177288.698082:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177288.698088:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.698094:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177288.698096:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.698098:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177288.698099:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177288.698103:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.698109:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177288.698113:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177288.698117:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177288.698120:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177288.698122:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.698123:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177288.698124:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.698126:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177288.698127:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177288.698130:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177288.698133:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177288.698135:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177288.698138:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177288.698144:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856360448:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177288.698148:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.698149:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177288.698152:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eded80 x1872934856360320/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.698158:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177288.698160:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177288.698162:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177288.698164:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177288.698166:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177288.698167:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177288.698168:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177288.698170:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0800.
02000000:00000001:3.0:1786177288.698172:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177288.698173:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177288.698175:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eded80.
00000100:00000001:3.0:1786177288.698177:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177288.698178:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.698180:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177288.698183:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd180 x1872934856360448/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.698188:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177288.698189:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177288.698190:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177288.698192:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177288.698193:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177288.698194:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177288.698196:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177288.698197:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:3.0:1786177288.698198:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177288.698200:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177288.698201:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd180.
00000100:00000001:3.0:1786177288.698203:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177288.698204:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.698205:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.698208:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177288.698210:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.698211:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177288.698213:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177288.698287:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177288.698293:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786177288.698297:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177288.698303:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177288.698311:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177288.698313:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177288.698317:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177288.698338:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177288.698345:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177288.698349:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177288.698350:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177288.698351:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177288.698354:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177288.698357:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7512
00000400:00000200:2.0:1786177288.698361:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177288.698364:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177288.698367:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177288.698369:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.698371:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177288.698374:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.698375:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177288.698378:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177288.698379:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177288.698381:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177288.698384:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786177288.698385:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786177288.698397:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177288.698401:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.698403:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177288.698404:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177288.698406:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177288.698408:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786177288.698410:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177288.698413:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177288.698415:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177288.698419:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177288.698421:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.698540:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177288.698543:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177288.698546:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.698547:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177288.698551:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000020:00000040:2.0:1786177288.698554:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177288.698557:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177288.698558:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177288.698560:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177288.698562:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177288.698564:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177288.698678:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786177288.698684:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177288.698689:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177288.698696:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177288.698699:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177288.698702:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177288.698710:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177288.698713:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7522)
00000100:00000001:3.0:1786177288.702104:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177288.702112:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177288.702114:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177288.702124:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702136:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177288.702139:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177288.702143:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177288.702146:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177288.702149:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177288.702152:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856360576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177288.702156:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177288.702161:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177288.702163:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177288.702165:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177288.702172:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177288.702174:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177288.702182:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177288.702187:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177288.702194:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177288.702199:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856360576, portal 4
00000100:00000001:3.0:1786177288.702202:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177288.702203:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178048 : -114548255373568 : ffff97d1a7edc700)
00000100:00000040:3.0:1786177288.702212:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702219:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177288.702221:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177288.702224:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177288.702228:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856360576, offset 0
00000400:00000200:3.0:1786177288.702234:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177288.702242:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177288.702248:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177288.702253:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177288.702257:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177288.702260:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177288.702263:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177288.702267:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254988:254988:256:4294967295] 192.168.202.110@tcp LPNI seq info [254977:254977:8:4294967295]
00000400:00000200:3.0:1786177288.702272:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177288.702282:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177288.702287:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177288.702293:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177288.702299:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177288.702303:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786177288.702307:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177288.702311:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1786177288.702315:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177288.702317:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177288.702322:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177288.702326:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177288.702331:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177288.702333:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177288.702337:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177288.702340:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702351:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177288.702355:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702361:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.702363:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177288.702369:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177288.702374:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177288.702377:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177288.702380:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302184462
00000100:00000001:3.0:1786177288.702383:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302188237
00000100:00000001:3.0:1786177288.702385:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302184462
00000100:00000001:3.0:1786177288.702387:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302188237
00000100:00000001:3.0:1786177288.702389:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177288.702392:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.702394:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177288.702399:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177288/real 1786177288]  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177288.702408:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177288.702409:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177288.702412:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177288.702415:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177288.702417:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177288.702419:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177288.702422:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177288.702430:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702436:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177288.702438:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.702440:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177288.702442:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177288.702445:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702452:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177288.702457:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702464:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177288.702466:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.702468:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177288.702469:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.702472:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177288.702474:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177288.702477:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177288.702481:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177288.702484:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177288.702488:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177288.702496:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856360576:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177288.702501:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177288.702504:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702532:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177288.702534:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177288.702537:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177288.702539:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177288.702541:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177288.702545:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856360704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177288.702548:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177288.702551:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177288.702552:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177288.702554:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177288.702559:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177288.702561:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177288.702564:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177288.702566:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177288.702570:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177288.702573:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856360704, portal 4
00000100:00000001:3.0:1786177288.702574:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177288.702575:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454180736 : -114548255370880 : ffff97d1a7edd180)
00000100:00000040:3.0:1786177288.702582:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702588:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177288.702591:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177288.702594:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177288.702596:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856360704, offset 0
00000400:00000200:3.0:1786177288.702600:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177288.702604:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177288.702608:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177288.702611:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177288.702613:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177288.702616:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177288.702618:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177288.702621:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254989:254989:256:4294967295] 192.168.202.110@tcp LPNI seq info [254978:254978:8:4294967295]
00000400:00000200:3.0:1786177288.702625:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177288.702631:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177288.702636:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177288.702640:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177288.702643:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177288.702646:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1786177288.702650:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177288.702652:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1786177288.702656:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177288.702657:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177288.702660:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177288.702663:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177288.702666:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177288.702667:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177288.702670:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177288.702673:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702680:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177288.702683:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702689:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.702691:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177288.702694:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177288.702698:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177288.702702:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177288.702704:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302184471
00000100:00000001:3.0:1786177288.702707:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302188237
00000100:00000001:3.0:1786177288.702709:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302184471
00000100:00000001:3.0:1786177288.702712:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302188237
00000100:00000001:3.0:1786177288.702714:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177288.702717:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.702719:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177288.702724:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177288/real 1786177288]  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177288.702733:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177288.702734:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177288.702736:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177288.702739:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177288.702741:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177288.702743:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177288.702747:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177288.702754:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702760:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177288.702761:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.702763:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177288.702764:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177288.702768:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702773:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177288.702777:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702783:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177288.702786:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.702787:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177288.702789:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.702790:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177288.702792:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177288.702794:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177288.702797:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177288.702800:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177288.702803:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177288.702810:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856360704:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177288.702814:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.702816:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177288.702820:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc700 x1872934856360576/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702826:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177288.702829:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177288.702832:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177288.702834:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177288.702837:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177288.702838:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177288.702841:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177288.702843:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:3.0:1786177288.702846:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177288.702849:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177288.702851:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc700.
00000100:00000001:3.0:1786177288.702854:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177288.702855:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.702857:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177288.702860:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd180 x1872934856360704/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177288.702866:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177288.702867:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177288.702868:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177288.702870:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177288.702871:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177288.702873:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177288.702874:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177288.702876:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:3.0:1786177288.702891:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177288.702892:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177288.702894:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd180.
00000100:00000001:3.0:1786177288.702896:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177288.702898:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.702900:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177288.702907:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177288.702908:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177288.702910:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177288.702912:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177289.272661:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177289.272671:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177289.272679:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177289.272681:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177289.272684:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177289.272686:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177289.272763:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177289.272771:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177289.272775:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177289.720827:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177289.720835:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177289.720837:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177289.720842:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177289.720845:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177289.720849:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177289.720851:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177289.720854:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177289.720857:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177290.296634:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177290.296638:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177290.296642:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177290.296643:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177290.296645:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177290.296646:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177290.296983:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177290.296987:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177290.296989:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177290.745099:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177290.745106:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177290.745108:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177290.745111:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177290.745114:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177290.745117:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177290.745119:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177290.745121:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177290.745123:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177291.320617:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177291.320624:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177291.320627:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177291.320630:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177291.320634:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177291.320637:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177291.320639:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177291.320641:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177291.320645:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177291.320646:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177291.320647:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177291.320649:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177291.320650:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177291.320652:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177291.320653:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177291.320655:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177291.320658:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177291.320659:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177291.320985:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177291.320988:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177291.320991:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177291.768646:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177291.768654:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177291.768656:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177291.768661:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177291.768664:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177291.768668:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177291.768671:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177291.768673:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177291.768676:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177292.344634:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177292.344641:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177292.344646:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177292.344647:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177292.344650:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177292.344651:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177292.344743:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177292.344746:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177292.344749:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177292.536625:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302188237, at 4302188288
00000100:00000010:1.0:1786177292.536639:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37f00.
00000100:00000040:0.0:1786177292.536712:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302188237, at 4302188288
00000100:00000010:0.0:1786177292.536717:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37cc0.
00000100:00000001:0.0:1786177292.792619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177292.792626:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177292.792628:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177292.792633:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177292.792636:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177292.792640:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177292.792643:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177292.792646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177292.792649:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177293.368644:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177293.368651:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177293.368657:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177293.368658:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177293.368661:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177293.368663:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177293.368719:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177293.368723:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177293.368726:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177293.816661:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1786177293.816666:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177293.816670:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177293.816674:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177293.816679:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177293.816682:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:0.0:1786177293.816683:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786177293.816689:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177293.816691:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177293.816697:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177293.816698:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786177293.816706:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177293.816713:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177293.816713:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7517
00000100:00000001:0.0:1786177293.816722:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177293.816733:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177293.816741:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177293.816745:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177293.816749:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177293.816753:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177293.816756:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177293.816760:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177293.816765:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177293.816768:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177293.816770:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177293.816774:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177293.816776:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177293.816779:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177293.816782:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177293.816785:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177293.816787:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177293.816789:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177293.816791:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177293.816795:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177293.816797:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177293.816799:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177293.816802:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177293.816804:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177293.816806:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177293.816808:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177293.816817:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786177293.816821:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177293.816824:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177293.816826:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177293.816858:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177293.816861:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177293.816865:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177293.816872:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786177293.816878:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177293.816886:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177293.816896:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177293.816900:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177293.816904:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177293.816925:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177293.816928:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177293.816931:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177293.816933:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177293.816934:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177293.816937:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177293.816940:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7517
00000400:00000200:2.0:1786177293.816944:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177293.816947:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177293.816949:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177293.816975:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177293.816981:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177293.816984:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177293.816995:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.817005:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177293.817007:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177293.817010:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177293.817012:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177293.817017:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177293.817018:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1786177293.817024:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177293.817028:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00100000:3.0:1786177293.817035:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856360832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177293.817036:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177293.817049:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177293.817052:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177293.817057:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177293.817060:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177293.817064:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177293.817072:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177293.817075:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177293.817083:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177293.817087:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177293.817095:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177293.817103:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856360832, portal 25
00000100:00000001:3.0:1786177293.817105:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177293.817107:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187904 : -114548255363712 : ffff97d1a7eded80)
00000020:00000040:2.0:1786177293.817110:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177293.817114:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000040:3.0:1786177293.817117:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177293.817118:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177293.817128:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177293.817138:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:3.0:1786177293.817142:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177293.817147:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:3.0:1786177293.817150:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177293.817158:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1786177293.817159:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177293.817167:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856360832, offset 0
00000100:00000040:2.0:1786177293.817168:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177293.817173:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177293.817179:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:2.0:1786177293.817180:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177293.817187:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177293.817190:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177293.817197:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177293.817198:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177293.817206:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177293.817209:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:2.0:1786177293.817216:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177293.817221:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786177293.817222:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177293.817228:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:3.0:1786177293.817229:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000008:00000001:2.0:1786177293.817238:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177293.817238:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177293.817245:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:2.0:1786177293.817246:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177293.817258:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000400:00000200:3.0:1786177293.817260:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254990:254990:256:4294967295] 192.168.202.110@tcp LPNI seq info [254979:254979:8:4294967295]
00000020:00000040:2.0:1786177293.817266:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000400:00000200:3.0:1786177293.817274:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177293.817278:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177293.817281:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177293.817285:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1786177293.817292:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177293.817300:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177293.817306:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177293.817324:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177293.817329:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786177293.817335:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177293.817339:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1786177293.817343:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177293.817346:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177293.817354:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177293.817359:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177293.817365:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177293.817367:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177293.817372:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177293.817376:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.817386:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177293.817391:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177293.817399:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177293.817400:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177293.817403:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.817405:0:2391:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:2.0:1786177293.817414:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:3.0:1786177293.817416:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177293.817420:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177293.817433:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177293.817442:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177293.817444:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177293.817447:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177293.817456:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177293.817459:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177293.817462:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177293.817464:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177293.817465:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177293.817467:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177293.817469:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7517
00000400:00000200:2.0:1786177293.817474:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177293.817477:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177293.817479:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177293.817482:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177293.817485:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177293.817486:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177293.817488:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177293.817490:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177293.817492:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177293.817493:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177293.817495:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177293.817497:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786177293.817500:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786177293.817503:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177293.817505:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177293.817506:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177293.817508:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177293.817546:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177293.817551:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302193358
00000020:00000040:2.0:1786177293.817558:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000010:3.0:1786177293.817559:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37a20.
00000100:00080000:2.0:1786177293.817571:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177293.817573:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177293.817575:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177293.817580:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177293.817584:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177293.817585:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302193358
02000000:00000001:2.0:1786177293.817588:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177293.817591:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37420.
00000100:00000001:2.0:1786177293.817592:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177293.817596:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1786177293.817598:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000008:00000001:2.0:1786177293.817603:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.817606:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177293.817609:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.817612:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177293.817615:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000100:00000001:3.0:1786177293.817617:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000020:00000040:2.0:1786177293.817623:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177293.817631:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:3.0:1786177293.817631:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177293/real 1786177293]  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177293.817638:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177293.817648:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177293.817653:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177293.817659:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1786177293.817665:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000010:2.0:1786177293.817667:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000400:00000001:3.0:1786177293.817670:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177293.817674:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00000001:2.0:1786177293.817675:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177293.817679:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177293.817683:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00100000:2.0:1786177293.817685:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177293.817688:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177293.817699:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177293.817703:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177293.817708:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177293.817724:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177293.817730:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.817773:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177293.817774:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.817778:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177293.817780:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177293.817782:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177293.817786:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177293.817788:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177293.817792:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177293.817795:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1786177293.817798:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177293.817801:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177293.817811:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7517
00000100:00000001:3.0:1786177293.817821:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1786177293.817824:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177293.817841:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1786177293.817847:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786177293.817848:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177293.817861:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177293.817869:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177293.817872:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:2.0:1786177293.817873:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177293.817882:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177293.817883:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177293.817891:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000001:2.0:1786177293.817892:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177293.817894:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786177293.817896:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177293.817901:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177293.817901:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177293.817907:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:2.0:1786177293.817909:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00080000:3.0:1786177293.817916:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000020:00000001:2.0:1786177293.817918:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177293.817925:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00080000:3.0:1786177293.817929:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000040:2.0:1786177293.817938:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177293.817939:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177293.817958:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177293.817968:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856360832:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177293.817973:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.817976:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177293.817979:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eded80 x1872934856360832/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.817993:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177293.817996:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177293.817999:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177293.818002:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177293.818005:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177293.818007:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177293.818009:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177293.818012:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:3.0:1786177293.818016:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177293.818018:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177293.818020:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eded80.
00000100:00000001:3.0:1786177293.818024:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177293.818025:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.818028:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.818036:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177293.818038:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.818040:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177293.818042:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177293.818043:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177293.818047:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.818054:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177293.818056:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177293.818058:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177293.818061:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177293.818063:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177293.818066:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856360960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177293.818070:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177293.818072:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177293.818074:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177293.818076:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177293.818079:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177293.818081:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177293.818086:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177293.818089:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177293.818093:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177293.818096:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856360960, portal 10
00000100:00000001:3.0:1786177293.818098:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177293.818099:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454181632 : -114548255369984 : ffff97d1a7edd500)
00000100:00000040:3.0:1786177293.818106:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.818113:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177293.818116:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177293.818117:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177293.818120:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856360960, offset 0
00000400:00000200:3.0:1786177293.818125:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177293.818129:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177293.818133:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177293.818136:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177293.818139:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177293.818141:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177293.818144:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177293.818148:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254991:254991:256:4294967295] 192.168.202.110@tcp LPNI seq info [254980:254980:8:4294967295]
00000400:00000200:3.0:1786177293.818152:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177293.818162:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177293.818168:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177293.818171:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177293.818176:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177293.818179:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786177293.818185:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177293.818188:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1786177293.818192:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177293.818195:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000100:00000001:2.0:1786177293.818196:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177293.818204:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1786177293.818207:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177293.818213:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177293.818217:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177293.818219:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177293.818222:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177293.818225:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.818233:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177293.818236:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.818244:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.818245:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177293.818248:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177293.818252:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177293.818255:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177293.818257:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302189578
00000100:00000001:3.0:1786177293.818260:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302193358
00000100:00000001:3.0:1786177293.818262:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302189578
00000100:00000001:3.0:1786177293.818264:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302193358
00000100:00000001:3.0:1786177293.818267:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177293.818269:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.818271:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177293.818274:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177293/real 1786177293]  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177293.818281:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177293.818283:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177293.818285:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177293.818287:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177293.818289:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177293.818291:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177293.818294:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177293.818303:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.818373:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177293.818375:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.818376:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177293.818378:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177293.818382:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.818389:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177293.818397:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177293.818400:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177293.818412:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000001:3.0:1786177293.818415:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177293.818422:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177293.818424:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177293.818429:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177293.818431:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177293.818435:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177293.818437:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177293.818439:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177293.818443:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177293.818445:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177293.818447:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177293.818452:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786177293.818455:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177293.818457:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177293.818458:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177293.818460:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177293.818462:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177293.818464:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177293.818468:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786177293.818472:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.818475:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177293.818478:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.818485:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177293.818487:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177293.818490:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177293.818495:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177293.818497:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177293.818503:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177293.818530:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856360960:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177293.818539:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177293.818544:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177293.818552:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177293.818552:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177293.818554:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177293.818561:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177293.818565:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177293.818573:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177293.818583:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00100000:3.0:1786177293.818584:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177293.818592:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.818594:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177293.818599:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177293.818604:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177293.818605:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177293.818608:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177293.818615:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177293.818621:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177293.818634:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177293.818640:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000001:2.0:1786177293.818641:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177293.818650:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1786177293.818651:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177293.818659:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856361088, portal 10
00000100:00080000:2.0:1786177293.818660:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177293.818666:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177293.818667:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177293.818670:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177293.818670:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000001:2.0:1786177293.818679:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177293.818687:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7517
00000100:00000040:3.0:1786177293.818687:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786177293.818702:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177293.818706:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177293.818714:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177293.818719:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000010:3.0:1786177293.818719:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00080000:2.0:1786177293.818732:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000200:3.0:1786177293.818732:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856361088, offset 0
00000100:00000040:2.0:1786177293.818743:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177293.818751:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177293.818753:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177293.818761:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177293.818763:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177293.818769:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:3.0:1786177293.818777:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000020:00000001:2.0:1786177293.818778:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177293.818783:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000400:00000200:3.0:1786177293.818787:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:2.0:1786177293.818788:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177293.818794:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000400:00000200:3.0:1786177293.818795:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:2.0:1786177293.818801:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:3.0:1786177293.818806:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000020:00000001:2.0:1786177293.818808:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:3.0:1786177293.818814:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000040:2.0:1786177293.818818:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177293.818824:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254992:254992:256:4294967295] 192.168.202.110@tcp LPNI seq info [254981:254981:8:4294967295]
00000100:00000001:2.0:1786177293.818826:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177293.818846:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177293.818854:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177293.818859:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177293.818863:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177293.818867:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177293.818870:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786177293.818875:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177293.818878:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1786177293.818882:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177293.818884:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177293.818887:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177293.818890:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177293.818893:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177293.818895:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177293.818898:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177293.818901:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.818910:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177293.818913:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.818919:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.818921:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177293.818924:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177293.818927:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177293.818930:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177293.818932:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302189704
00000100:00000001:3.0:1786177293.818934:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302193358
00000100:00000001:3.0:1786177293.818936:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302189704
00000100:00000001:3.0:1786177293.818938:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302193358
00000100:00000001:3.0:1786177293.818940:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177293.818942:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.818944:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177293.818948:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177293/real 1786177293]  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177293.818959:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177293.818960:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177293.818962:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177293.818965:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177293.818966:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177293.818968:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177293.818972:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177293.818980:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.818986:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177293.818988:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.818989:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177293.818991:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177293.818994:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819001:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177293.819004:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819011:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177293.819012:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.819014:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177293.819015:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.819017:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177293.819018:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177293.819021:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177293.819024:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177293.819027:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177293.819031:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177293.819038:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361088:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177293.819045:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177293.819049:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177293.819050:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177293.819058:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd500 x1872934856360960/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177293.819061:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177293.819071:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:3.0:1786177293.819081:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00080000:2.0:1786177293.819086:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:3.0:1786177293.819086:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177293.819091:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
00000100:00000001:2.0:1786177293.819092:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177293.819098:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000001:2.0:1786177293.819099:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000020:00000001:3.0:1786177293.819102:0:2391:0:(genops.c:1238:class_import_put()) Process entered
02000000:00000001:2.0:1786177293.819106:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1786177293.819106:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177293.819113:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000001:2.0:1786177293.819115:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177293.819118:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786177293.819119:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177293.819124:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000001:3.0:1786177293.819127:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000008:00000001:2.0:1786177293.819129:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177293.819131:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000001:2.0:1786177293.819135:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177293.819135:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd500.
00000100:00000001:3.0:1786177293.819145:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000010:2.0:1786177293.819146:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000100:00000001:3.0:1786177293.819148:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1786177293.819152:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177293.819153:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1786177293.819161:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:3.0:1786177293.819162:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856361088/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177293.819166:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177293.819173:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177293.819179:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177293.819185:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.819186:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177293.819191:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177293.819192:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
02000000:00000010:3.0:1786177293.819193:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177293.819203:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:2.0:1786177293.819205:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177293.819207:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177293.819214:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177293.819220:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
00000100:00100000:2.0:1786177293.819222:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:3.0:1786177293.819223:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:3.0:1786177293.819231:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177293.819235:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177293.819239:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:3.0:1786177293.819244:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177293.819247:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.819256:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.819265:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177293.819268:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177293.819271:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177293.819276:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177293.819277:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.819283:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177293.819288:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177293.819293:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819301:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177293.819303:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177293.819305:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1786177293.819352:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1786177293.819363:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177293.819365:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177293.819368:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177293.819373:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7527)
00000100:00100000:3.0:1786177293.819375:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177293.819381:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177293.819384:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177293.819386:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177293.819388:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177293.819392:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177293.819395:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177293.819400:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177293.819403:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177293.819407:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177293.819410:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856361216, portal 4
00000100:00000001:3.0:1786177293.819412:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177293.819413:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000040:3.0:1786177293.819420:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819477:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177293.819480:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177293.819483:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177293.819486:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856361216, offset 0
00000400:00000200:3.0:1786177293.819490:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177293.819494:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177293.819497:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177293.819500:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177293.819503:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177293.819505:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177293.819508:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177293.819534:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254993:254993:256:4294967295] 192.168.202.110@tcp LPNI seq info [254982:254982:8:4294967295]
00000400:00000200:3.0:1786177293.819539:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177293.819545:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177293.819549:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177293.819552:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177293.819557:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177293.819559:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786177293.819563:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177293.819566:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1786177293.819570:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177293.819572:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177293.819575:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177293.819578:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177293.819581:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177293.819583:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177293.819585:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177293.819588:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819597:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177293.819600:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819606:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.819608:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177293.819611:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177293.819614:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177293.819617:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177293.819619:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302189580
00000100:00000001:3.0:1786177293.819621:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302193358
00000100:00000001:3.0:1786177293.819623:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302189580
00000100:00000001:3.0:1786177293.819626:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302193358
00000100:00000001:3.0:1786177293.819628:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177293.819630:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.819632:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177293.819635:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177293/real 1786177293]  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177293.819643:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177293.819644:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177293.819646:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177293.819648:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177293.819650:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177293.819652:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177293.819655:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177293.819663:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819669:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177293.819671:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.819672:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177293.819674:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177293.819677:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819683:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177293.819687:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819694:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177293.819696:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.819698:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177293.819699:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.819701:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177293.819703:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177293.819705:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177293.819709:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177293.819711:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177293.819715:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177293.819722:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361216:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177293.819726:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.819728:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177293.819731:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856361216/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819737:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177293.819738:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177293.819740:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177293.819743:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177293.819744:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177293.819746:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177293.819748:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177293.819750:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:3.0:1786177293.819752:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177293.819753:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177293.819755:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:3.0:1786177293.819757:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177293.819759:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.819761:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.819764:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177293.819765:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.819768:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177293.819769:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177293.819771:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177293.819774:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819781:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177293.819782:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177293.819785:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177293.819786:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177293.819788:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177293.819791:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361344:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177293.819794:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177293.819796:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177293.819797:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177293.819799:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177293.819802:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177293.819804:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177293.819807:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177293.819809:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177293.819813:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177293.819815:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856361344, portal 4
00000100:00000001:3.0:1786177293.819817:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177293.819818:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:3.0:1786177293.819824:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819844:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177293.819846:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177293.819849:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177293.819852:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856361344, offset 0
00000400:00000200:3.0:1786177293.819855:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177293.819859:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177293.819862:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177293.819865:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177293.819868:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177293.819870:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177293.819872:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177293.819875:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254994:254994:256:4294967295] 192.168.202.110@tcp LPNI seq info [254983:254983:8:4294967295]
00000400:00000200:3.0:1786177293.819879:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177293.819885:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177293.819889:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177293.819892:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177293.819896:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177293.819899:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1786177293.819903:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177293.819906:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1786177293.819909:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177293.819911:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177293.819913:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177293.819917:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177293.819920:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177293.819922:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177293.819923:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177293.819927:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819935:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177293.819938:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.819944:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.819946:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177293.819949:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177293.819952:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177293.819955:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177293.819956:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302189588
00000100:00000001:3.0:1786177293.819960:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302193358
00000100:00000001:3.0:1786177293.819962:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302189588
00000100:00000001:3.0:1786177293.819964:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302193358
00000100:00000001:3.0:1786177293.819966:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177293.819968:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.819970:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177293.819973:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177293/real 1786177293]  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177293.819981:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177293.820328:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177293.820331:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177293.820334:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177293.820336:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177293.820338:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177293.820342:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177293.820351:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.820357:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177293.820359:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.820361:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177293.820362:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177293.820366:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.820372:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177293.820376:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.820383:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177293.820386:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.820388:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177293.820389:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.820391:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177293.820393:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177293.820395:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177293.820398:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177293.820400:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177293.820404:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177293.820411:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361344:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177293.820414:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.820416:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177293.820419:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856361344/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177293.820434:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177293.820436:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177293.820437:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177293.820440:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177293.820441:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177293.820443:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177293.820445:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177293.820447:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:3.0:1786177293.820449:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177293.820450:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177293.820452:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:3.0:1786177293.820454:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177293.820456:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.820457:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177293.820461:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177293.820462:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177293.820464:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177293.820466:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177294.392677:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177294.392686:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177294.392694:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177294.392695:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177294.392698:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177294.392702:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177294.392705:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177294.392710:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177294.392713:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177294.392716:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177294.392718:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177294.392721:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177294.392877:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177294.392882:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177294.392885:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177294.840692:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177294.840703:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177294.840706:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177294.840712:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177294.840716:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177294.840721:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177294.840724:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177294.840728:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177294.840731:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177295.416664:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177295.416673:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177295.416680:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177295.416681:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177295.416685:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177295.416686:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177295.416798:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177295.416802:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177295.416806:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177295.864869:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177295.864923:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177295.864926:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177295.864933:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177295.864938:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177295.864943:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177295.864946:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177295.864949:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177295.864953:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177296.440655:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177296.440662:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177296.440668:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177296.440669:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177296.440672:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177296.440673:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177296.440764:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177296.440771:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177296.440777:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177296.888665:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177296.888676:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177296.888679:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177296.888686:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177296.888690:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177296.888695:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177296.888699:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177296.888703:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177296.888706:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177297.464701:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177297.464719:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177297.464727:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177297.464729:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177297.464732:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177297.464733:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177297.464816:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177297.464822:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177297.464826:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177297.656652:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302193358, at 4302193408
00000100:00000040:1.0:1786177297.656652:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302193358, at 4302193408
00000100:00000010:1.0:1786177297.656671:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37420.
00000100:00000010:0.0:1786177297.656675:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37a20.
00000800:00000200:2.0:1786177297.784942:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786177297.784954:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786177297.784960:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:1.0:1786177297.785890:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1786177297.785898:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1786177297.785913:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1786177297.785919:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1786177297.785922:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177297.912781:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177297.912802:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177297.912805:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177297.912813:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177297.912817:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177297.912822:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177297.912825:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177297.912828:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177297.912832:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177298.488705:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177298.488714:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177298.488722:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177298.488723:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177298.488727:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177298.488728:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177298.488795:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177298.488798:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177298.488801:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786177298.936656:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:0.0:1786177298.936664:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1786177298.936670:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177298.936672:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177298.936674:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177298.936676:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177298.936680:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:0.0:1786177298.936683:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:2.0:1786177298.936688:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1786177298.936689:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177298.936694:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177298.936703:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177298.936707:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177298.936709:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7522
00000100:00000001:0.0:1786177298.936713:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177298.936721:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177298.936734:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177298.936739:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177298.936743:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177298.936747:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177298.936750:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177298.936753:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177298.936757:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177298.936760:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177298.936764:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177298.936767:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177298.936770:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177298.936773:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177298.936775:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177298.936777:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177298.936778:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177298.936789:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177298.936792:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177298.936794:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177298.936797:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177298.936800:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177298.936803:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177298.936805:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177298.936807:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177298.936811:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177298.936827:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786177298.936832:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177298.936837:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177298.936839:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177298.936843:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177298.936846:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177298.936851:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177298.936863:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786177298.936871:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177298.936882:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177298.936891:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177298.936895:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177298.936899:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177298.936924:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177298.936927:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177298.936929:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177298.936930:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177298.936932:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177298.936933:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177298.936935:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7522
00000400:00000200:2.0:1786177298.936938:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177298.936940:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177298.936942:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177298.936944:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177298.936946:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177298.936947:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177298.936948:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177298.936950:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177298.936950:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177298.936952:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177298.936953:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177298.936954:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177298.936956:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177298.936957:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177298.936959:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177298.936960:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177298.936962:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177298.936964:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177298.936966:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177298.936967:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177298.936969:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177298.936975:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177298.936976:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:2.0:1786177298.936977:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177298.936980:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177298.936984:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000008:00100000:2.0:1786177298.936986:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1786177298.936988:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000008:00000001:2.0:1786177298.936991:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177298.936995:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177298.937000:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786177298.937001:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786177298.937006:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177298.937012:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177298.937014:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.937017:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177298.937018:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177298.937020:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177298.937022:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177298.937025:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177298.937026:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000010:2.0:1786177298.937030:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
02000000:00000001:3.0:1786177298.937031:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177298.937037:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177298.937037:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177298.937043:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856361472:192.168.202.110@tcp:250:kworker.0
00000100:00100000:2.0:1786177298.937047:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177298.937050:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177298.937057:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1786177298.937059:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1786177298.937062:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177298.937063:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177298.937068:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:2.0:1786177298.937071:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1786177298.937076:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:2.0:1786177298.937078:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177298.937080:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177298.937081:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177298.937085:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177298.937086:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1786177298.937089:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:2.0:1786177298.937090:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177298.937095:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177298.937097:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7522
00000400:00000010:3.0:1786177298.937097:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000200:2.0:1786177298.937102:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177298.937105:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177298.937106:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1786177298.937108:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000040:2.0:1786177298.937109:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177298.937113:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177298.937114:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177298.937116:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000200:3.0:1786177298.937120:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856361472, portal 25
00000100:00000001:3.0:1786177298.937124:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177298.937125:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000020:00000001:2.0:1786177298.937131:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177298.937134:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000100:00000040:3.0:1786177298.937134:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177298.937138:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177298.937142:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177298.937147:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786177298.937153:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:3.0:1786177298.937153:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177298.937159:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177298.937162:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177298.937164:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1786177298.937165:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000040:2.0:1786177298.937166:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177298.937171:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177298.937174:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000200:3.0:1786177298.937174:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856361472, offset 0
00000100:00080000:2.0:1786177298.937177:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177298.937180:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177298.937181:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177298.937183:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177298.937186:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177298.937187:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177298.937188:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177298.937191:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177298.937194:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177298.937195:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177298.937198:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786177298.937200:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:3.0:1786177298.937201:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177298.937202:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177298.937204:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177298.937207:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177298.937208:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177298.937209:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177298.937213:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000400:00000200:3.0:1786177298.937213:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:2.0:1786177298.937219:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177298.937224:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:2.0:1786177298.937225:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177298.937233:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:2.0:1786177298.937234:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177298.937237:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177298.937241:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177298.937243:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177298.937248:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177298.937252:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254995:254995:256:4294967295] 192.168.202.110@tcp LPNI seq info [254984:254984:8:4294967295]
00000400:00000200:3.0:1786177298.937259:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177298.937266:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177298.937269:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177298.937277:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000400:00000200:3.0:1786177298.937279:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786177298.937280:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177298.937284:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177298.937288:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:3.0:1786177298.937292:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786177298.937294:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7522
00000400:00000200:2.0:1786177298.937301:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000800:00000010:3.0:1786177298.937302:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000400:00000200:2.0:1786177298.937305:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177298.937309:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177298.937314:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:3.0:1786177298.937315:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177298.937318:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177298.937322:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:3.0:1786177298.937323:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1786177298.937325:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177298.937331:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177298.937333:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000800:00000200:3.1:1786177298.937335:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000001:2.0:1786177298.937336:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177298.937340:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000800:00000200:3.1:1786177298.937342:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:2.0:1786177298.937344:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177298.937349:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000100:3.0:1786177298.937352:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177298.937356:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177298.937363:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177298.937368:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177298.937374:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177298.937377:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177298.937381:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177298.937385:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937395:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177298.937399:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937406:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.937408:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177298.937415:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177298.937419:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177298.937423:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302198477
00000100:00000010:3.0:1786177298.937429:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37180.
00000100:00000001:3.0:1786177298.937436:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177298.937443:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302198477
00000100:00000010:3.0:1786177298.937445:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad379c0.
00000100:00000001:3.0:1786177298.937448:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177298.937452:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177298.937454:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.937457:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177298.937461:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177298/real 1786177298]  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177298.937470:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177298.937471:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177298.937474:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177298.937477:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177298.937479:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177298.937482:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177298.937485:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177298.937493:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937499:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177298.937501:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.937503:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177298.937505:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177298.937509:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937544:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177298.937549:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937562:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177298.937564:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.937566:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177298.937567:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.937570:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177298.937572:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177298.937575:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177298.937578:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177298.937581:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177298.937584:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177298.937592:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856361472:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177298.937596:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.937599:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177298.937601:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856361472/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937608:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177298.937610:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177298.937613:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177298.937617:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177298.937618:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177298.937620:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177298.937623:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177298.937625:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:3.0:1786177298.937628:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177298.937630:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177298.937632:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:3.0:1786177298.937635:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177298.937636:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.937638:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.937646:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177298.937648:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.937650:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177298.937652:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177298.937653:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177298.937657:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937663:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177298.937665:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177298.937667:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177298.937668:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177298.937670:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177298.937673:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361600:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177298.937676:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177298.937678:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177298.937680:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177298.937681:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177298.937685:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177298.937687:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177298.937690:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177298.937693:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177298.937697:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177298.937700:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856361600, portal 10
00000100:00000001:3.0:1786177298.937702:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177298.937703:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:3.0:1786177298.937710:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937717:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177298.937718:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177298.937720:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177298.937722:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856361600, offset 0
00000400:00000200:3.0:1786177298.937724:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177298.937728:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177298.937731:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177298.937733:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177298.937735:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177298.937737:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177298.937738:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177298.937740:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254996:254996:256:4294967295] 192.168.202.110@tcp LPNI seq info [254985:254985:8:4294967295]
00000400:00000200:3.0:1786177298.937742:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177298.937746:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177298.937750:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177298.937753:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177298.937757:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177298.937759:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786177298.937764:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177298.937767:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1786177298.937771:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177298.937773:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177298.937776:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177298.937801:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177298.937804:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177298.937806:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177298.937809:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177298.937812:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937820:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177298.937823:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937830:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.937831:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177298.937835:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177298.937839:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177298.937842:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177298.937844:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302194697
00000100:00000001:3.0:1786177298.937847:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302198477
00000100:00000001:3.0:1786177298.937850:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302194697
00000100:00000001:3.0:1786177298.937852:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302198477
00000100:00000001:3.0:1786177298.937854:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177298.937856:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.937858:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177298.937873:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177298/real 1786177298]  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177298.937884:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177298.937885:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177298.937887:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177298.937890:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177298.937891:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177298.937893:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177298.937897:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177298.937905:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937911:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177298.937913:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.937914:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177298.937916:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177298.937919:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937926:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177298.937930:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937937:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177298.937939:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.937941:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177298.937942:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.937944:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177298.937946:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177298.937949:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177298.937952:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177298.937954:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177298.937958:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177298.937965:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361600:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177298.937969:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177298.937972:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.937978:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177298.937979:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177298.937982:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177298.937983:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177298.937985:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177298.937988:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177298.937991:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177298.937993:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177298.937994:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177298.937996:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177298.937999:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177298.938001:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177298.938004:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177298.938006:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177298.938010:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177298.938013:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856361728, portal 10
00000100:00000001:3.0:1786177298.938015:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177298.938016:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000040:3.0:1786177298.938022:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.938028:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177298.938030:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177298.938032:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177298.938034:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856361728, offset 0
00000400:00000200:3.0:1786177298.938038:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177298.938042:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177298.938045:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177298.938048:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177298.938050:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177298.938053:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177298.938055:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177298.938057:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254997:254997:256:4294967295] 192.168.202.110@tcp LPNI seq info [254986:254986:8:4294967295]
00000400:00000200:3.0:1786177298.938061:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177298.938068:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177298.938072:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177298.938076:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177298.938080:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177298.938083:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786177298.938087:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177298.938090:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1786177298.938094:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177298.938095:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177298.938099:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177298.938102:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177298.938105:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177298.938106:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177298.938109:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177298.938112:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.938125:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177298.938128:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.938134:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.938136:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177298.938139:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177298.938142:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177298.938145:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177298.938147:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302194822
00000100:00000001:3.0:1786177298.938150:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302198477
00000100:00000001:3.0:1786177298.938151:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302194822
00000100:00000001:3.0:1786177298.938153:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302198477
00000100:00000001:3.0:1786177298.938154:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177298.938156:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.938157:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177298.938160:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177298/real 1786177298]  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177298.938165:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177298.938167:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177298.938168:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177298.938170:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177298.938171:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177298.938173:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177298.938174:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177298.938180:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.938184:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177298.938185:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.938187:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177298.938188:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177298.938192:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177298.938196:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177298.938199:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177298.938205:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177298.938207:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177298.938210:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177298.938213:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177298.938214:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177298.938215:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177298.938217:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177298.938220:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177298.938220:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177298.938223:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:2.0:1786177298.938227:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177298.938227:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177298.938231:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177298.938234:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00080000:2.0:1786177298.938235:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000040:3.0:1786177298.938238:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177298.938239:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177298.938244:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:3.0:1786177298.938248:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361728:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177298.938250:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.938252:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177298.938254:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856361600/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177298.938255:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177298.938260:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177298.938265:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177298.938266:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177298.938268:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000008:00100000:2.0:1786177298.938269:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000010:3.0:1786177298.938270:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177298.938274:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000008:00000001:2.0:1786177298.938276:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177298.938277:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177298.938280:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:2.0:1786177298.938281:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177298.938282:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177298.938284:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
00000100:00000010:2.0:1786177298.938289:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
02000000:00000001:3.0:1786177298.938289:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177298.938290:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177298.938293:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786177298.938295:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177298.938296:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177298.938297:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.938301:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1786177298.938303:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:3.0:1786177298.938304:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856361728/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177298.938306:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177298.938312:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177298.938314:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177298.938316:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177298.938316:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177298.938320:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:2.0:1786177298.938322:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177298.938322:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177298.938324:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177298.938327:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177298.938328:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177298.938329:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000010:2.0:1786177298.938331:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
02000000:00000001:3.0:1786177298.938331:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177298.938332:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177298.938335:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:3.0:1786177298.938338:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177298.938339:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.938339:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.938340:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.938347:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177298.938349:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177298.938350:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177298.938351:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177298.938353:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177298.938363:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177298.938366:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177298.938371:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177298.938396:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177298.938400:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177298.938403:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177298.938405:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177298.938408:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177298.938411:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177298.938413:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7522
00000400:00000200:2.0:1786177298.938418:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177298.938422:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177298.938425:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177298.938428:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177298.938430:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177298.938431:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177298.938435:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1786177298.938436:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177298.938439:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786177298.938441:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:3.0:1786177298.938446:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1786177298.938447:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177298.938450:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177298.938455:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177298.938457:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1786177298.938460:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000001:3.0:1786177298.938460:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177298.938466:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
02000000:00000001:3.0:1786177298.938466:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177298.938468:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:2.0:1786177298.938472:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
02000000:00000001:3.0:1786177298.938473:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177298.938480:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00100000:3.0:1786177298.938481:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177298.938486:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177298.938488:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:2.0:1786177298.938492:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177298.938493:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177298.938496:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177298.938497:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000001:3.0:1786177298.938501:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:2.0:1786177298.938502:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786177298.938507:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000010:3.0:1786177298.938507:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000001:2.0:1786177298.938528:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177298.938531:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177298.938532:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1786177298.938537:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177298.938538:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:3.0:1786177298.938542:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
02000000:00000001:2.0:1786177298.938544:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177298.938547:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:2.0:1786177298.938548:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000200:3.0:1786177298.938551:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856361856, portal 4
00000008:00100000:2.0:1786177298.938552:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177298.938555:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000008:00000001:2.0:1786177298.938558:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.938558:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000100:00000001:2.0:1786177298.938564:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177298.938567:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786177298.938571:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000100:00000001:3.0:1786177298.938572:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:2.0:1786177298.938577:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000040:3.0:1786177298.938577:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177298.938581:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
02000000:00000001:2.0:1786177298.938582:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:3.0:1786177298.938584:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856361856, offset 0
02000000:00000001:2.0:1786177298.938585:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177298.938590:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177298.938591:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177298.938594:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1786177298.938595:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177298.938598:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177298.938600:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177298.938604:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177298.938605:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
02000000:00000010:2.0:1786177298.938606:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000400:00000200:3.0:1786177298.938608:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1786177298.938612:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177298.938612:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177298.938616:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254998:254998:256:4294967295] 192.168.202.110@tcp LPNI seq info [254987:254987:8:4294967295]
00000400:00000200:3.0:1786177298.938621:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00100000:2.0:1786177298.938622:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1786177298.938632:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1786177298.938641:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000200:3.0:1786177298.938642:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177298.938647:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1786177298.938647:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177298.938652:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177298.938657:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000100:00000040:2.0:1786177298.938658:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1786177298.938665:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177298.938670:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1786177298.938677:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177298.938679:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1786177298.938679:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177298.938684:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1786177298.938688:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7532)
00000400:00000200:3.0:1786177298.938691:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177298.938695:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177298.938697:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177298.938700:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177298.938704:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.938712:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177298.938715:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.938721:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.938722:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177298.938726:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177298.938729:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177298.938732:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177298.938734:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302194699
00000100:00000001:3.0:1786177298.938737:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302198477
00000100:00000001:3.0:1786177298.938739:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302194699
00000100:00000001:3.0:1786177298.938741:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302198477
00000100:00000001:3.0:1786177298.938744:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177298.938746:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.938747:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177298.938751:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177298/real 1786177298]  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177298.938758:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177298.938759:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177298.938761:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177298.938762:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177298.938763:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177298.938765:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177298.938767:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177298.938774:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.938797:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177298.938799:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.938800:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177298.938802:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177298.938805:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.938812:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177298.938816:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.938823:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177298.938826:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.938828:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177298.938829:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.938832:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177298.938833:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177298.938836:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177298.938839:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177298.938841:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177298.938845:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177298.938853:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361856:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177298.938856:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.938858:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177298.938861:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856361856/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.938867:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177298.938869:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177298.938871:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177298.938879:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177298.938881:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177298.938882:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177298.938884:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177298.938886:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:3.0:1786177298.938889:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177298.938890:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177298.938891:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
00000100:00000001:3.0:1786177298.938893:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177298.938895:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.938896:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.938900:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177298.938901:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.938903:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177298.938904:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177298.938906:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177298.938910:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.938917:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177298.938918:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177298.938920:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177298.938922:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177298.938924:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177298.938928:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177298.938930:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177298.938932:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177298.938934:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177298.938936:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177298.938939:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177298.938941:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177298.938944:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177298.938946:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177298.938950:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177298.938953:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856361984, portal 4
00000100:00000001:3.0:1786177298.938955:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177298.938956:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178944 : -114548255372672 : ffff97d1a7edca80)
00000100:00000040:3.0:1786177298.938962:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.938968:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177298.938970:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177298.938971:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177298.938973:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856361984, offset 0
00000400:00000200:3.0:1786177298.938976:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177298.938979:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177298.938982:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177298.938984:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177298.938986:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177298.938987:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177298.938989:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177298.938991:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [254999:254999:256:4294967295] 192.168.202.110@tcp LPNI seq info [254988:254988:8:4294967295]
00000400:00000200:3.0:1786177298.938993:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177298.938997:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177298.939000:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177298.939002:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177298.939005:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177298.939007:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1786177298.939009:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177298.939011:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1786177298.939013:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177298.939014:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177298.939016:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177298.939018:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177298.939020:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177298.939021:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177298.939023:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177298.939025:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.939030:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177298.939032:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.939036:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.939037:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177298.939040:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177298.939042:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177298.939044:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177298.939045:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302194707
00000100:00000001:3.0:1786177298.939047:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302198477
00000100:00000001:3.0:1786177298.939049:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302194707
00000100:00000001:3.0:1786177298.939050:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302198477
00000100:00000001:3.0:1786177298.939051:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177298.939052:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.939053:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177298.939055:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177298/real 1786177298]  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177298.939060:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177298.939061:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177298.939062:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177298.939064:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177298.939065:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177298.939066:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177298.939068:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177298.939072:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.939075:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177298.939076:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.939077:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177298.939078:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177298.939080:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.939084:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177298.939086:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.939090:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177298.939091:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.939092:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177298.939093:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.939094:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177298.939095:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177298.939096:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177298.939097:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177298.939098:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177298.939100:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177298.939108:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856361984:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177298.939112:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.939114:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177298.939117:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edca80 x1872934856361984/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177298.939122:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177298.939124:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177298.939126:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177298.939128:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177298.939129:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177298.939131:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177298.939133:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177298.939135:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:3.0:1786177298.939137:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177298.939139:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177298.939140:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edca80.
00000100:00000001:3.0:1786177298.939143:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177298.939144:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.939146:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177298.939149:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177298.939150:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177298.939151:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177298.939153:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177299.512804:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177299.512814:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177299.512823:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177299.512824:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177299.512828:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177299.512829:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177299.512952:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177299.512956:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177299.512960:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177299.960686:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177299.960696:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177299.960699:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177299.960707:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177299.960712:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177299.960716:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177299.960720:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177299.960723:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177299.960727:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177300.536651:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177300.536658:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177300.536663:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177300.536663:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177300.536671:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177300.536672:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177300.536741:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177300.536746:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177300.536749:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177300.984669:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177300.984680:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177300.984682:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177300.984690:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177300.984694:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177300.984699:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177300.984702:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177300.984705:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177300.984709:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177301.560662:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177301.560683:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177301.560686:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177301.560690:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177301.560696:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177301.560699:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177301.560702:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177301.560704:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177301.560709:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177301.560711:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177301.560712:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177301.560714:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177301.560715:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177301.560717:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177301.560718:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177301.560720:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177301.560724:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177301.560725:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177301.562600:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177301.562607:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177301.562613:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177302.008664:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177302.008674:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177302.008676:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177302.008682:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177302.008687:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177302.008691:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177302.008695:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177302.008698:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177302.008701:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177302.584680:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177302.584689:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177302.584696:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177302.584697:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177302.584701:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177302.584702:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177302.584816:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177302.584820:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177302.584824:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177302.776654:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302198477, at 4302198528
00000100:00000040:1.0:1786177302.776654:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302198477, at 4302198528
00000100:00000010:1.0:1786177302.776674:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad379c0.
00000100:00000010:0.0:1786177302.776676:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37180.
00000100:00000001:0.0:1786177303.032668:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177303.032678:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177303.032681:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177303.032688:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177303.032693:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177303.032698:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177303.032701:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177303.032705:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177303.032708:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177303.608726:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177303.608870:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177303.608879:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177303.608880:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177303.608884:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177303.608888:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177303.608890:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177303.608894:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177303.608898:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177303.608901:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177303.608904:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177303.608906:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177303.609598:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177303.609603:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177303.609607:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177304.056725:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1786177304.056876:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177304.056889:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177304.056892:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1786177304.056911:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177304.056914:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177304.056920:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177304.056925:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177304.056929:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177304.056933:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177304.056936:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177304.056939:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177304.057030:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177304.057035:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177304.057044:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7527
00000400:00000200:2.0:1786177304.057054:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177304.057063:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177304.057067:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177304.057071:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057074:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177304.057077:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057080:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177304.057084:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177304.057087:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177304.057090:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177304.057093:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177304.057096:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177304.057099:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177304.057103:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057105:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177304.057107:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057109:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177304.057113:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177304.057116:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057119:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177304.057123:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177304.057127:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177304.057130:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.057132:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177304.057135:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177304.057146:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf800.
00000020:00000040:2.0:1786177304.057151:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177304.057156:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177304.057158:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.057162:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177304.057166:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177304.057170:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177304.057180:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786177304.057188:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177304.057200:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177304.057211:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177304.057216:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177304.057221:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000069f0ae06] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177304.057294:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177304.057298:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177304.057301:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177304.057303:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177304.057304:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177304.057308:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177304.057311:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7527
00000400:00000200:2.0:1786177304.057315:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177304.057319:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177304.057321:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177304.057323:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057325:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177304.057327:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057329:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177304.057331:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177304.057333:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177304.057334:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177304.057336:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177304.057338:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177304.057340:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177304.057343:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057345:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177304.057346:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057348:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177304.057351:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177304.057354:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057357:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177304.057359:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177304.057365:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177304.057368:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.057370:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177304.057375:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1786177304.057375:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177304.057381:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000008:00000001:2.0:1786177304.057382:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.057386:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1786177304.057387:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177304.057397:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.057407:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177304.057409:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177304.057424:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177304.057427:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177304.057431:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177304.057435:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856362112:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177304.057440:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177304.057445:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177304.057449:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177304.057452:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177304.057458:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177304.057460:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177304.057466:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177304.057471:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177304.057480:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177304.057489:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856362112, portal 25
00000100:00000001:3.0:1786177304.057491:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177304.057495:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454190592 : -114548255361024 : ffff97d1a7edf800)
00000100:00000010:2.0:1786177304.057496:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edf100.
00000020:00000040:2.0:1786177304.057502:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000040:3.0:1786177304.057503:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177304.057508:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177304.057532:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.057535:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177304.057536:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177304.057538:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177304.057544:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786177304.057548:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177304.057555:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177304.057560:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177304.057568:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057570:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177304.057573:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177304.057574:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000040:2.0:1786177304.057580:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bb9724bd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177304.057581:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856362112, offset 0
00000100:00000001:2.0:1786177304.057594:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177304.057595:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:2.0:1786177304.057598:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177304.057602:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177304.057604:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177304.057606:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177304.057608:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177304.057612:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7527
00000400:00000200:2.0:1786177304.057616:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177304.057619:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177304.057621:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177304.057624:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057626:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177304.057628:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057630:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177304.057633:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177304.057634:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177304.057635:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177304.057637:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177304.057640:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786177304.057744:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786177304.057750:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057752:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177304.057754:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057756:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177304.057758:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786177304.057761:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057764:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177304.057766:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177304.057769:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177304.057771:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.057773:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177304.057774:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177304.057776:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.057778:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177304.057784:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000020:00000040:2.0:1786177304.057786:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177304.057788:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177304.057790:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.057791:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177304.057793:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177304.057795:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177304.057800:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786177304.057804:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177304.057809:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177304.057817:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177304.057819:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177304.057823:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177304.057830:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177304.057832:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177304.057835:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177304.057837:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177304.057839:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177304.057842:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177304.057844:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7527
00000400:00000200:2.0:1786177304.057849:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177304.057852:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177304.057854:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177304.057857:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057859:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177304.057861:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057863:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177304.057865:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177304.057867:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177304.057868:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177304.057870:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177304.057872:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177304.057874:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786177304.057877:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057878:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177304.057880:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057881:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177304.057883:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177304.057885:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057887:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177304.057889:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177304.057892:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177304.057893:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.057895:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177304.057897:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177304.057899:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.057901:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177304.057904:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000020:00000040:2.0:1786177304.057906:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177304.057908:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177304.057910:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.057911:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177304.057912:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177304.057914:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177304.057918:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786177304.057921:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177304.057925:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177304.057932:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177304.057934:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177304.057938:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177304.057945:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177304.057948:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177304.057952:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177304.057953:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177304.057955:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177304.057957:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177304.057960:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7527
00000400:00000200:2.0:1786177304.057964:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177304.057967:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177304.057969:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177304.057971:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057973:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177304.057975:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057977:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177304.057978:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177304.057980:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177304.057981:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177304.057984:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786177304.057986:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786177304.057988:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177304.057991:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057993:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177304.057995:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177304.057997:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177304.057999:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786177304.058001:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177304.058004:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177304.058006:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177304.058008:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177304.058010:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.058012:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177304.058015:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177304.058017:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.058018:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177304.058022:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786177304.058023:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177304.058026:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177304.058027:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177304.058028:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177304.058029:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177304.058031:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177304.058034:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1800.
00000100:00000001:2.0:1786177304.058037:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177304.058041:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177304.058048:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177304.058050:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177304.058054:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177304.058060:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177304.058065:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7537)
00000400:00000200:3.0:1786177304.058219:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177304.058227:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177304.058233:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177304.058238:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177304.058242:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177304.058245:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177304.058249:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255000:255000:256:4294967295] 192.168.202.110@tcp LPNI seq info [254989:254989:8:4294967295]
00000400:00000200:3.0:1786177304.058253:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177304.058265:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177304.058272:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177304.058278:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177304.058287:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177304.058291:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786177304.058297:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177304.058301:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786177304.058305:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177304.058307:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177304.058315:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177304.058321:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177304.058327:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177304.058330:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177304.058335:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177304.058339:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.058350:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177304.058354:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.058361:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.058363:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177304.058371:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177304.058375:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177304.058378:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302203598
00000100:00000010:3.0:1786177304.058384:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad374e0.
00000100:00000001:3.0:1786177304.058391:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177304.058397:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302203598
00000100:00000010:3.0:1786177304.058401:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37120.
00000100:00000001:3.0:1786177304.058405:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177304.058408:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177304.058422:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.058425:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177304.058430:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177304/real 1786177304]  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177304.058446:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177304.058448:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177304.058451:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177304.058454:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177304.058456:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177304.058459:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177304.058463:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177304.058471:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.058479:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177304.058480:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.058482:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177304.058485:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177304.058489:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.058496:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177304.058500:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.058508:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177304.058533:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.058535:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177304.058536:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.058539:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177304.058541:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177304.058544:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177304.058548:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177304.058551:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177304.058556:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177304.058565:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856362112:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177304.058569:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.058571:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177304.058574:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf800 x1872934856362112/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.058581:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177304.058583:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177304.058586:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177304.058590:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177304.058592:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177304.058594:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177304.058708:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177304.058712:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:3.0:1786177304.058718:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177304.058720:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177304.058722:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf800.
00000100:00000001:3.0:1786177304.058725:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177304.058726:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.058739:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.058749:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177304.058752:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.058755:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177304.058757:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177304.058758:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177304.058764:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.058771:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177304.058773:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177304.058776:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177304.058778:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177304.058780:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177304.058784:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856362240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177304.058788:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177304.058791:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177304.058792:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177304.058794:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177304.058799:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177304.059305:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177304.059311:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177304.059315:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177304.059321:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177304.059325:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856362240, portal 10
00000100:00000001:3.0:1786177304.059328:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177304.059330:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454188800 : -114548255362816 : ffff97d1a7edf100)
00000100:00000040:3.0:1786177304.059338:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059345:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177304.059347:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177304.059350:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177304.059352:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856362240, offset 0
00000400:00000200:3.0:1786177304.059357:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177304.059363:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177304.059368:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177304.059371:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177304.059374:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177304.059377:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177304.059380:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177304.059383:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255001:255001:256:4294967295] 192.168.202.110@tcp LPNI seq info [254990:254990:8:4294967295]
00000400:00000200:3.0:1786177304.059387:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177304.059393:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177304.059398:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177304.059402:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177304.059407:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177304.059469:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786177304.059477:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177304.059481:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786177304.059485:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177304.059487:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177304.059490:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177304.059494:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177304.059498:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177304.059500:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177304.059503:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177304.059507:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059537:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177304.059541:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059548:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.059550:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177304.059555:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177304.059560:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177304.059563:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177304.059566:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302199820
00000100:00000001:3.0:1786177304.059568:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302203598
00000100:00000001:3.0:1786177304.059571:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302199820
00000100:00000001:3.0:1786177304.059573:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302203598
00000100:00000001:3.0:1786177304.059575:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177304.059577:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.059579:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177304.059583:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177304/real 1786177304]  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177304.059592:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177304.059594:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177304.059596:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177304.059599:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177304.059601:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177304.059603:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177304.059607:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177304.059615:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059622:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177304.059623:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.059625:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177304.059627:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177304.059631:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059637:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177304.059641:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059648:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177304.059651:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.059653:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177304.059654:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.059656:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177304.059658:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177304.059661:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177304.059664:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177304.059667:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177304.059671:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177304.059679:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856362240:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177304.059683:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177304.059687:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059694:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177304.059695:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177304.059697:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177304.059700:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177304.059702:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177304.059705:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856362368:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177304.059708:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177304.059710:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177304.059712:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177304.059714:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177304.059718:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177304.059720:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177304.059723:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177304.059726:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177304.059741:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177304.059744:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856362368, portal 10
00000100:00000001:3.0:1786177304.059746:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177304.059748:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178944 : -114548255372672 : ffff97d1a7edca80)
00000100:00000040:3.0:1786177304.059754:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059760:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177304.059763:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177304.059766:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177304.059768:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856362368, offset 0
00000400:00000200:3.0:1786177304.059773:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177304.059776:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177304.059780:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177304.059783:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177304.059786:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177304.059788:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177304.059791:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177304.059793:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255002:255002:256:4294967295] 192.168.202.110@tcp LPNI seq info [254991:254991:8:4294967295]
00000400:00000200:3.0:1786177304.059806:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177304.059812:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177304.059817:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177304.059820:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177304.059824:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177304.059827:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786177304.059831:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177304.059833:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786177304.059838:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177304.059839:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177304.059842:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177304.059846:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177304.059849:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177304.059851:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177304.059853:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177304.059857:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059865:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177304.059867:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059875:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.059876:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177304.059879:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177304.059882:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177304.059885:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177304.059887:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302199945
00000100:00000001:3.0:1786177304.059890:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302203598
00000100:00000001:3.0:1786177304.059893:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302199945
00000100:00000001:3.0:1786177304.059896:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302203598
00000100:00000001:3.0:1786177304.059898:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177304.059900:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.059901:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177304.059905:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177304/real 1786177304]  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177304.059913:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177304.059915:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177304.059917:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177304.059919:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177304.059921:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177304.059923:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177304.059926:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177304.059933:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059940:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177304.059941:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.059943:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177304.059944:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177304.059948:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059955:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177304.059959:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.059966:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177304.059968:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.059969:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177304.059971:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.059973:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177304.059975:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177304.059977:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177304.059981:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177304.059983:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177304.059987:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177304.059994:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856362368:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177304.059998:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177304.060002:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.060009:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177304.060011:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177304.060013:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177304.060015:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177304.060017:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177304.060021:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856362496:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177304.060024:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177304.060026:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177304.060027:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177304.060029:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177304.060032:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177304.060034:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177304.060038:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177304.060040:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177304.060044:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177304.060047:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856362496, portal 4
00000100:00000001:3.0:1786177304.060048:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177304.060050:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000100:00000040:3.0:1786177304.060056:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.060062:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177304.060065:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177304.060067:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177304.060070:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856362496, offset 0
00000400:00000200:3.0:1786177304.060074:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177304.060078:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177304.060081:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177304.060084:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177304.060086:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177304.060089:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177304.060091:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177304.060094:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255003:255003:256:4294967295] 192.168.202.110@tcp LPNI seq info [254992:254992:8:4294967295]
00000400:00000200:3.0:1786177304.060098:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177304.060104:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177304.060108:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177304.060112:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177304.060116:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177304.060118:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786177304.060122:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177304.060124:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786177304.060129:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177304.060131:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177304.060134:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177304.060137:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177304.060140:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177304.060141:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177304.060144:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177304.060147:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.060155:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177304.060158:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.060169:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.060171:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177304.060174:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177304.060177:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177304.060181:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177304.060183:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302199820
00000100:00000001:3.0:1786177304.060185:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302203598
00000100:00000001:3.0:1786177304.060187:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302199820
00000100:00000001:3.0:1786177304.060292:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302203598
00000100:00000001:3.0:1786177304.060296:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177304.060298:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.060300:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177304.060305:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177304/real 1786177304]  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177304.060314:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177304.060315:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177304.060316:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177304.060319:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177304.060321:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177304.060323:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177304.060326:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177304.060334:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.060341:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177304.060343:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.060344:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177304.060346:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177304.060350:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.060357:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177304.060361:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.060368:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177304.060370:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.060371:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177304.060373:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.060375:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177304.060376:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177304.060379:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177304.060382:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177304.060384:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177304.060388:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177304.060396:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856362496:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177304.060401:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177304.060405:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.060914:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177304.060917:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177304.060921:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177304.060922:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177304.060925:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177304.060929:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856362624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177304.060932:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177304.060935:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177304.060937:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177304.060939:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177304.060943:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177304.060946:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177304.060949:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177304.060952:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177304.060956:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177304.060959:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856362624, portal 4
00000100:00000001:3.0:1786177304.060961:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177304.060962:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000040:3.0:1786177304.060969:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.060976:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177304.060978:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177304.060980:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177304.060983:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856362624, offset 0
00000400:00000200:3.0:1786177304.060986:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177304.060991:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177304.060995:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177304.060998:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177304.061000:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177304.061003:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177304.061005:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177304.061008:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255004:255004:256:4294967295] 192.168.202.110@tcp LPNI seq info [254993:254993:8:4294967295]
00000400:00000200:3.0:1786177304.061013:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177304.061019:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177304.061023:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177304.061026:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177304.061031:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177304.061033:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1786177304.061037:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177304.061040:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1786177304.061044:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177304.061047:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177304.061050:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177304.061054:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177304.061057:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177304.061059:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177304.061062:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177304.061065:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.061074:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177304.061076:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177329 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.061083:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.061084:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177304.061087:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177304.061090:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177304.061093:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177304.061095:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302199829
00000100:00000001:3.0:1786177304.061098:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302203598
00000100:00000001:3.0:1786177304.061100:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302199829
00000100:00000001:3.0:1786177304.061102:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302203598
00000100:00000001:3.0:1786177304.061104:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177304.061106:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.061108:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177304.061112:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177304/real 1786177304]  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177304.061118:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177304.061120:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177304.061121:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177304.061124:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177304.061125:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177304.061128:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177304.061131:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177304.061140:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.061147:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177304.061148:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.061150:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177304.061152:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177304.061155:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.061162:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177304.061166:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.061173:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177304.061175:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.061177:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177304.061178:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.061180:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177304.061182:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177304.061185:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177304.061253:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177304.061256:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177304.061261:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177304.061269:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856362624:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177304.061274:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.061277:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177304.061281:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edf100 x1872934856362240/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.061288:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177304.061290:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177304.061293:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177304.061297:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177304.061299:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177304.061301:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177304.061303:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177304.061306:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:3.0:1786177304.061310:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177304.061312:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177304.061314:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edf100.
00000100:00000001:3.0:1786177304.061317:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177304.061319:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.061321:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177304.061324:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edca80 x1872934856362368/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.061330:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177304.061332:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177304.061333:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177304.061335:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177304.061336:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177304.061338:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177304.061339:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177304.061341:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:3.0:1786177304.061343:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177304.061344:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177304.061345:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edca80.
00000100:00000001:3.0:1786177304.061348:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177304.061349:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.061350:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177304.061353:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856362496/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.061359:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177304.061362:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177304.061363:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177304.061366:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177304.061367:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177304.061368:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177304.061371:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177304.061372:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:3.0:1786177304.061374:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177304.061375:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177304.061377:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
00000100:00000001:3.0:1786177304.061379:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177304.061380:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.061382:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177304.061385:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856362624/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177329 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177304.061391:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177304.061392:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177304.061394:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:3.0:1786177304.061396:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177304.061398:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177304.061399:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177304.061401:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177304.061402:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1800.
02000000:00000001:3.0:1786177304.061404:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177304.061405:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177304.061407:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:3.0:1786177304.061409:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177304.061923:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.061927:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177304.061935:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177304.061937:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177304.061940:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177304.061942:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177304.632675:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177304.632685:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177304.632692:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177304.632694:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177304.632697:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177304.632698:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177304.632852:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177304.632857:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177304.632861:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177305.080625:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177305.080633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177305.080635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177305.080640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177305.080644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177305.080648:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177305.080651:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177305.080653:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177305.080656:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177305.656615:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177305.656621:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177305.656626:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177305.656628:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177305.656631:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177305.656632:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177305.656695:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177305.656700:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177305.656703:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177306.104611:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177306.104616:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177306.104617:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177306.104621:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177306.104624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177306.104626:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177306.104628:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177306.104630:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177306.104632:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177306.680637:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177306.680645:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177306.680651:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177306.680652:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177306.680655:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177306.680656:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177306.681750:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177306.681753:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177306.681755:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177307.128620:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177307.128627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177307.128629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177307.128634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177307.128637:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177307.128641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177307.128644:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177307.128647:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177307.128649:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177307.704666:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177307.704676:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177307.704683:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177307.704684:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177307.704687:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177307.704688:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177307.705612:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177307.705616:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177307.705619:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177307.896741:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302203598, at 4302203648
00000100:00000010:1.0:1786177307.896762:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37120.
00000100:00000040:0.0:1786177307.896866:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302203598, at 4302203648
00000100:00000010:0.0:1786177307.896874:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad374e0.
00000100:00000001:0.0:1786177308.152670:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177308.152681:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177308.152683:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177308.152701:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177308.152706:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177308.152711:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177308.152714:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177308.152719:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177308.152722:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177308.728714:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177308.728725:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177308.728732:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177308.728733:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177308.728737:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177308.728738:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177308.728905:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177308.728910:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177308.728913:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177309.176600:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177309.176610:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177309.176612:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177309.176620:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177309.176624:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177309.176629:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177309.176633:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177309.176636:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177309.176639:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177309.176655:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177309.176662:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177309.176665:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177309.176669:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177309.176674:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177309.176718:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7532
00000400:00000200:3.0:1786177309.176731:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177309.176739:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177309.176742:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177309.176746:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.176749:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177309.176752:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.176755:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177309.176759:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177309.176762:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177309.176765:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177309.176769:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177309.176772:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177309.176774:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177309.176777:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.176779:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177309.176780:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.176782:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177309.176785:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177309.176789:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177309.176791:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177309.176795:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177309.176799:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177309.176802:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.176806:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177309.176808:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177309.176819:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69fb80.
00000020:00000040:3.0:1786177309.176902:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177309.176909:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177309.176911:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.176914:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177309.176918:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177309.176925:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177309.176934:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786177309.176941:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177309.176952:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177309.176963:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177309.176968:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177309.176972:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014f55bbb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177309.177001:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177309.177005:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177309.177009:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177309.177010:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177309.177012:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177309.177015:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177309.177018:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7532
00000400:00000200:3.0:1786177309.177022:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177309.177025:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177309.177028:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177309.177030:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177033:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177309.177035:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177037:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177309.177040:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177309.177041:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177309.177043:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177309.177045:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786177309.177047:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786177309.177049:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177309.177052:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177054:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177309.177056:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177058:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177309.177061:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786177309.177063:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177065:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177309.177067:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177309.177072:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177309.177075:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.177076:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177309.177081:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177309.177084:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.177086:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177309.177090:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d880.
00000020:00000040:3.0:1786177309.177092:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177309.177094:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177309.177095:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.177097:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177309.177098:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177309.177100:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177309.177103:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:3.0:1786177309.177106:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177309.177111:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177309.177118:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177309.177120:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177309.177124:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097f9b8db] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177309.177130:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177309.177133:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177309.177136:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177309.177137:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177309.177138:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177309.177141:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177309.177143:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7532
00000400:00000200:3.0:1786177309.177146:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177309.177149:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177309.177152:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177309.177155:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177157:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177309.177158:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177160:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177309.177162:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177309.177163:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177309.177165:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177309.177166:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786177309.177168:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177309.177170:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177309.177173:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177175:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177309.177176:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177178:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177309.177180:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786177309.177183:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177186:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177309.177187:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177309.177190:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177309.177192:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.177194:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177309.177197:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177309.177199:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.177201:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177309.177204:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c380.
00000020:00000040:3.0:1786177309.177206:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177309.177209:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177309.177210:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.177212:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177309.177213:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177309.177215:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177309.177218:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:3.0:1786177309.177221:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177309.177226:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177309.177233:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177309.177235:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177309.177239:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000833bf1c9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177309.177247:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177309.177250:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177309.177253:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177309.177255:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177309.177257:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177309.177259:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177309.177262:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7532
00000400:00000200:3.0:1786177309.177265:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177309.177268:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177309.177270:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177309.177274:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177276:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177309.177278:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177280:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177309.177282:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177309.177283:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177309.177285:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177309.177286:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177309.177288:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177309.177291:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177309.177294:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177296:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177309.177298:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177300:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177309.177301:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177309.177303:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177305:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177309.177307:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177309.177309:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177309.177311:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.177312:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177309.177315:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177309.177317:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.177319:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177309.177322:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ed80.
00000020:00000040:3.0:1786177309.177324:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177309.177326:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177309.177328:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.177329:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177309.177331:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177309.177333:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177309.177336:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000001:3.0:1786177309.177338:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177309.177342:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177309.177349:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177309.177351:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177309.177354:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ccd6c6d2] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177309.177362:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177309.177364:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177309.177367:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177309.177368:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177309.177370:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177309.177372:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177309.177374:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7532
00000400:00000200:3.0:1786177309.177377:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177309.177379:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177309.177381:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177309.177384:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177386:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177309.177388:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177390:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177309.177392:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177309.177393:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177309.177395:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177309.177397:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177309.177399:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177309.177400:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177309.177404:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177406:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177309.177408:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177410:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177309.177412:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786177309.177414:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177309.177415:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177309.177417:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177309.177419:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177309.177421:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.177423:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177309.177425:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177309.177427:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.177429:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177309.177431:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e680.
00000020:00000040:3.0:1786177309.177433:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177309.177436:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177309.177437:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177309.177439:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177309.177440:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177309.177441:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177309.177444:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:3.0:1786177309.177447:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177309.177451:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177309.177458:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177309.177460:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177309.177463:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1839131] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177309.177471:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177309.177477:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7542)
00000100:00000001:2.0:1786177309.177549:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177309.177554:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177309.177556:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177309.177564:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.177573:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177309.177575:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177309.177578:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177309.177581:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177309.177583:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177309.177587:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856362752:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177309.177592:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177309.177596:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177309.177599:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177309.177603:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177309.177609:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab0800.
02000000:00000001:2.0:1786177309.177611:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177309.177616:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177309.177621:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177309.177628:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177309.177636:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856362752, portal 25
00000100:00000001:2.0:1786177309.177639:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177309.177641:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000040:2.0:1786177309.177646:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.177653:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177309.177656:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177309.177658:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177309.177662:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856362752, offset 0
00000400:00000200:2.0:1786177309.177669:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177309.177677:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177309.177702:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177309.177709:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177309.177714:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177309.177718:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177309.177721:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177309.177726:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255005:255005:256:4294967295] 192.168.202.110@tcp LPNI seq info [254994:254994:8:4294967295]
00000400:00000200:2.0:1786177309.177730:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177309.177741:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177309.177747:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177309.177753:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177309.177761:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177309.177765:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1786177309.177770:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177309.177774:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1786177309.177778:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177309.177781:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177309.177789:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177309.177794:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177309.177799:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177309.177802:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177309.177806:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177309.177810:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.177819:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177309.177822:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.177953:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177309.177955:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177309.177964:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177309.177968:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177309.177971:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302208718
00000100:00000010:2.0:1786177309.177978:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25360.
00000100:00000001:2.0:1786177309.177985:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177309.177991:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302208718
00000100:00000010:2.0:1786177309.177994:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25d20.
00000100:00000001:2.0:1786177309.177997:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786177309.178000:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177309.178004:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177309.178006:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177309.178011:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177309/real 1786177309]  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177309.178020:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177309.178022:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177309.178024:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177309.178027:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177309.178028:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177309.178031:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177309.178034:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177309.178043:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178049:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177309.178051:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.178053:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177309.178055:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177309.178059:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178066:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177309.178071:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178079:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177309.178082:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.178083:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177309.178085:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.178087:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177309.178090:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177309.178093:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177309.178098:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177309.178101:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177309.178105:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177309.178113:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856362752:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177309.178118:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177309.178122:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178128:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177309.178129:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177309.178131:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177309.178132:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177309.178134:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177309.178137:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856362880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177309.178140:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177309.178143:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177309.178144:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177309.178146:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177309.178150:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177309.178153:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177309.178156:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177309.178158:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177309.178161:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177309.178164:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856362880, portal 10
00000100:00000001:2.0:1786177309.178165:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177309.178167:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
00000100:00000040:2.0:1786177309.178173:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178179:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177309.178180:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177309.178182:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177309.178185:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856362880, offset 0
00000400:00000200:2.0:1786177309.178189:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177309.178192:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177309.178196:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177309.178199:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177309.178201:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177309.178204:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177309.178206:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177309.178209:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255006:255006:256:4294967295] 192.168.202.110@tcp LPNI seq info [254995:254995:8:4294967295]
00000400:00000200:2.0:1786177309.178213:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177309.178219:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177309.178223:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177309.178226:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177309.178230:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177309.178233:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1786177309.178237:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177309.178240:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1786177309.178245:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177309.178246:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177309.178250:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177309.178253:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177309.178256:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177309.178257:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177309.178259:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177309.178263:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178270:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177309.178273:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178279:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177309.178281:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177309.178284:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177309.178287:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:2.0:1786177309.178291:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177309.178293:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302204938
00000100:00000001:2.0:1786177309.178296:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302208718
00000100:00000001:2.0:1786177309.178298:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302204938
00000100:00000001:2.0:1786177309.178300:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302208718
00000100:00000001:2.0:1786177309.178302:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177309.178304:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177309.178306:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177309.178310:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177309/real 1786177309]  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177309.178317:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177309.178318:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177309.178320:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177309.178322:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177309.178323:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177309.178325:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177309.178328:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177309.178337:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178344:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177309.178345:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.178347:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177309.178349:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177309.178352:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178359:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177309.178363:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178370:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177309.178372:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.178374:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177309.178375:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.178377:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177309.178380:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177309.178382:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177309.178385:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177309.178388:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177309.178392:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177309.178400:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856362880:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177309.178404:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177309.178407:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178414:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177309.178415:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177309.178418:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177309.178419:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177309.178422:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177309.178425:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177309.178428:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177309.178430:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177309.178431:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177309.178433:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177309.178438:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1800.
02000000:00000001:2.0:1786177309.178440:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177309.178443:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177309.178446:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177309.178448:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177309.178451:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856363008, portal 10
00000100:00000001:2.0:1786177309.178453:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177309.178454:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764293504 : -114547945258112 : ffff97d1ba69c380)
00000100:00000040:2.0:1786177309.178459:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178467:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177309.178469:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177309.178471:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177309.178474:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856363008, offset 0
00000400:00000200:2.0:1786177309.178477:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177309.178481:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177309.178484:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177309.178487:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177309.178490:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177309.178492:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177309.178495:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177309.178498:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255007:255007:256:4294967295] 192.168.202.110@tcp LPNI seq info [254996:254996:8:4294967295]
00000400:00000200:2.0:1786177309.178502:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177309.178508:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177309.178543:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177309.178546:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177309.178550:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177309.178553:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1786177309.178557:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177309.178560:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1786177309.178563:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177309.178565:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177309.178569:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177309.178572:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177309.178575:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177309.178577:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177309.178579:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177309.178582:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178590:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177309.178593:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178606:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177309.178607:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177309.178610:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177309.178613:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:2.0:1786177309.178616:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177309.178618:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302205064
00000100:00000001:2.0:1786177309.178620:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302208718
00000100:00000001:2.0:1786177309.178623:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302205064
00000100:00000001:2.0:1786177309.178625:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302208718
00000100:00000001:2.0:1786177309.178627:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177309.178629:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177309.178630:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177309.178634:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177309/real 1786177309]  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177309.178641:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177309.178643:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177309.178644:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177309.178646:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177309.178648:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177309.178650:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177309.178653:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177309.178661:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178668:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177309.178670:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.178671:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177309.178673:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177309.178676:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178696:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177309.178700:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178707:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177309.178709:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.178711:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177309.178713:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.178714:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177309.178716:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177309.178719:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177309.178722:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177309.178725:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177309.178729:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177309.178737:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363008:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177309.178740:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177309.178743:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178751:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177309.178752:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177309.178755:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177309.178757:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177309.178759:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177309.178762:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177309.178765:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177309.178767:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177309.178769:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177309.178770:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177309.178774:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177309.178776:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177309.178779:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177309.178782:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177309.178786:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177309.178788:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856363136, portal 4
00000100:00000001:2.0:1786177309.178790:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177309.178791:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:2.0:1786177309.178797:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178804:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177309.178806:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177309.178808:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177309.178811:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856363136, offset 0
00000400:00000200:2.0:1786177309.178815:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177309.178819:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177309.178822:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177309.178838:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177309.178841:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177309.178843:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177309.178846:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177309.178849:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255008:255008:256:4294967295] 192.168.202.110@tcp LPNI seq info [254997:254997:8:4294967295]
00000400:00000200:2.0:1786177309.178854:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177309.178860:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177309.178865:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177309.178868:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177309.178872:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177309.178875:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1786177309.178879:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177309.178881:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1786177309.178885:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177309.178887:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177309.178890:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177309.178893:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177309.178896:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177309.178898:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177309.178900:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177309.178904:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178911:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177309.178914:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178921:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177309.178922:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177309.178925:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177309.178928:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:2.0:1786177309.178931:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177309.178933:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302204939
00000100:00000001:2.0:1786177309.178935:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302208718
00000100:00000001:2.0:1786177309.178937:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302204939
00000100:00000001:2.0:1786177309.178939:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302208718
00000100:00000001:2.0:1786177309.178941:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177309.178942:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177309.178944:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177309.178947:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177309/real 1786177309]  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177309.178954:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177309.178955:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177309.178956:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177309.178959:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177309.178960:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177309.178962:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177309.178965:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177309.178973:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178978:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177309.178980:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.178982:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177309.178983:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177309.178987:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.178995:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177309.178998:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179006:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177309.179008:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.179010:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177309.179011:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.179013:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177309.179014:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177309.179017:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177309.179020:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177309.179022:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177309.179026:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177309.179034:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363136:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177309.179038:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177309.179041:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179048:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177309.179049:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177309.179051:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177309.179053:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177309.179055:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177309.179057:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177309.179061:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177309.179062:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177309.179064:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177309.179066:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177309.179069:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3800.
02000000:00000001:2.0:1786177309.179071:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177309.179073:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177309.179076:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177309.179079:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177309.179082:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856363264, portal 4
00000100:00000001:2.0:1786177309.179083:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177309.179084:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00000040:2.0:1786177309.179090:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179095:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177309.179097:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177309.179099:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177309.179101:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856363264, offset 0
00000400:00000200:2.0:1786177309.179105:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177309.179109:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177309.179112:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177309.179114:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177309.179116:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177309.179118:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177309.179120:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177309.179123:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255009:255009:256:4294967295] 192.168.202.110@tcp LPNI seq info [254998:254998:8:4294967295]
00000400:00000200:2.0:1786177309.179126:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177309.179132:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177309.179137:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177309.179140:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177309.179143:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177309.179146:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1786177309.179149:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177309.179150:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1786177309.179153:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177309.179155:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177309.179157:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177309.179160:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177309.179163:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177309.179165:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177309.179167:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177309.179170:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179176:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177309.179178:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177334 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179184:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177309.179185:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177309.179187:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177309.179190:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:2.0:1786177309.179193:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177309.179195:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302204947
00000100:00000001:2.0:1786177309.179197:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302208718
00000100:00000001:2.0:1786177309.179200:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302204947
00000100:00000001:2.0:1786177309.179202:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302208718
00000100:00000001:2.0:1786177309.179204:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177309.179206:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177309.179208:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177309.179211:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177309/real 1786177309]  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177309.179218:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177309.179219:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177309.179220:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177309.179222:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177309.179223:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177309.179225:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177309.179227:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177309.179235:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179241:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177309.179242:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.179244:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177309.179245:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177309.179247:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179251:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177309.179254:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179261:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177309.179263:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.179265:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177309.179266:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.179267:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177309.179268:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177309.179270:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177309.179273:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177309.179275:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177309.179278:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177309.179285:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363264:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177309.179289:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.179291:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177309.179295:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856362752/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179302:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177309.179304:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177309.179307:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab0800.
02000000:00000001:2.0:1786177309.179311:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177309.179313:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177309.179315:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1786177309.179318:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177309.179320:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:2.0:1786177309.179324:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177309.179326:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177309.179327:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:2.0:1786177309.179330:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177309.179331:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.179333:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177309.179335:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856362880/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179342:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177309.179343:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177309.179345:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177309.179347:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177309.179348:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177309.179349:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177309.179350:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177309.179352:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:2.0:1786177309.179354:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177309.179355:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177309.179356:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:2.0:1786177309.179358:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177309.179359:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.179360:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177309.179363:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c380 x1872934856363008/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179370:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177309.179371:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177309.179373:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1800.
02000000:00000001:2.0:1786177309.179374:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177309.179375:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177309.179376:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177309.179378:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177309.179379:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:2.0:1786177309.179380:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177309.179381:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177309.179382:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c380.
00000100:00000001:2.0:1786177309.179384:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177309.179385:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.179387:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177309.179389:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856363136/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179395:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177309.179396:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177309.179398:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177309.179400:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177309.179402:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177309.179403:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177309.179405:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177309.179406:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:2.0:1786177309.179408:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177309.179408:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177309.179409:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:2.0:1786177309.179412:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177309.179413:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.179414:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177309.179417:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856363264/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177334 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177309.179423:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177309.179424:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177309.179425:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3800.
02000000:00000001:2.0:1786177309.179427:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177309.179428:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177309.179429:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177309.179430:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177309.179431:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:2.0:1786177309.179432:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177309.179433:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177309.179434:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:2.0:1786177309.179436:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177309.179436:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.179439:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177309.179447:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177309.179449:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177309.179451:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177309.179453:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177309.752986:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177309.752997:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177309.753004:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177309.753006:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177309.753009:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177309.753010:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177309.753182:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177309.753186:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177309.753191:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177310.200638:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177310.200646:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177310.200648:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177310.200654:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177310.200658:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177310.200661:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177310.200665:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177310.200668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177310.200684:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177310.776958:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177310.776967:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177310.776973:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177310.776975:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177310.776978:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177310.776979:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177310.777185:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177310.777190:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177310.777194:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177311.224608:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177311.224619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177311.224633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177311.224640:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177311.224644:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177311.224649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177311.224652:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177311.224656:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177311.224658:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177311.800898:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177311.800910:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177311.800913:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177311.800916:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177311.800922:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177311.800925:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177311.800927:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177311.800930:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177311.800934:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177311.800936:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177311.800937:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177311.800939:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177311.800940:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177311.800942:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177311.800944:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177311.800946:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177311.800949:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177311.800951:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177311.801198:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177311.801202:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177311.801204:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177312.248692:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177312.248703:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177312.248706:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177312.248714:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177312.248718:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177312.248724:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177312.248728:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177312.248732:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177312.248735:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177312.824805:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177312.824815:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177312.824822:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177312.824825:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177312.824828:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177312.824829:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177312.824896:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177312.824901:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177312.824905:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177313.016657:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302208718, at 4302208768
00000100:00000040:1.0:1786177313.016657:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302208718, at 4302208768
00000100:00000010:0.0:1786177313.016676:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25360.
00000100:00000010:1.0:1786177313.016679:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25d20.
00000100:00000001:0.0:1786177313.272729:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177313.272741:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177313.272743:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177313.272751:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177313.272756:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177313.272759:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177313.272762:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177313.272765:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177313.272768:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177313.848726:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177313.848733:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177313.848738:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177313.848739:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177313.848742:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177313.848746:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177313.848749:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177313.848752:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177313.848755:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177313.848757:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177313.848759:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177313.848761:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177313.848859:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177313.848863:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177313.848867:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786177314.296662:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177314.296682:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177314.296684:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177314.296688:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177314.296693:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177314.296701:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7537
00000400:00000200:0.0:1786177314.296710:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177314.296716:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177314.296720:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177314.296723:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.296726:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177314.296728:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.296731:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177314.296731:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000020:00000001:0.0:1786177314.296737:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:3.0:1786177314.296738:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000020:00000040:0.0:1786177314.296740:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000100:00000001:3.0:1786177314.296741:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177314.296746:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00100000:3.0:1786177314.296747:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000020:00000040:0.0:1786177314.296750:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000100:00000001:3.0:1786177314.296752:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1786177314.296755:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000001:3.0:1786177314.296757:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1786177314.296761:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000001:3.0:1786177314.296762:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177314.296767:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177314.296768:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.296772:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177314.296772:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177314.296776:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.296780:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177314.296783:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177314.296787:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177314.296790:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177314.296793:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177314.296796:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177314.296799:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.296802:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177314.296804:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177314.296814:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b800.
00000020:00000040:0.0:1786177314.296818:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177314.296822:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177314.296824:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.296827:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177314.296829:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177314.296834:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177314.296841:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7800.
00000100:00000001:0.0:1786177314.296847:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177314.296855:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177314.296865:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177314.296870:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177314.296874:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000192ee9a0] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177314.296902:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177314.296906:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177314.296910:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177314.296912:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177314.296913:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177314.296916:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177314.296919:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7537
00000400:00000200:0.0:1786177314.296923:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177314.296926:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177314.296928:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177314.296931:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.296933:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177314.296935:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.296936:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177314.296939:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177314.296940:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177314.296941:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177314.296943:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177314.296945:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177314.296947:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177314.296950:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.296952:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177314.296954:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.296956:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177314.296958:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177314.296960:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177314.296963:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177314.296965:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177314.296970:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177314.296973:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.296975:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177314.296979:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177314.296982:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.296984:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177314.296988:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418000.
00000020:00000040:0.0:1786177314.296990:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177314.296992:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177314.296993:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.296995:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177314.296997:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177314.296998:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177314.297002:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6000.
00000100:00000001:0.0:1786177314.297005:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177314.297010:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177314.297017:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177314.297019:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177314.297023:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002e10e4f5] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177314.297031:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177314.297034:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177314.297038:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177314.297039:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177314.297041:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177314.297043:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177314.297046:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7537
00000400:00000200:0.0:1786177314.297049:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177314.297052:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177314.297055:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177314.297058:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297059:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177314.297061:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297063:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177314.297065:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177314.297066:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177314.297068:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177314.297070:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177314.297071:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177314.297074:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177314.297076:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297090:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177314.297092:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297094:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177314.297097:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786177314.297100:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297102:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177314.297103:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177314.297106:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177314.297108:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.297110:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177314.297112:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177314.297114:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.297116:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177314.297119:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a300.
00000020:00000040:0.0:1786177314.297122:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177314.297124:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177314.297125:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.297127:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177314.297128:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177314.297130:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177314.297134:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000001:0.0:1786177314.297137:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177314.297141:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177314.297148:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177314.297150:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177314.297161:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b0d3569] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177314.297169:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177314.297171:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177314.297174:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177314.297176:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177314.297177:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177314.297180:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177314.297183:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7537
00000400:00000200:0.0:1786177314.297186:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177314.297189:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177314.297191:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177314.297193:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297196:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177314.297198:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297199:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177314.297201:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177314.297202:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177314.297204:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177314.297206:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177314.297208:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177314.297210:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177314.297213:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297214:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177314.297216:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297218:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177314.297221:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177314.297223:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297225:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177314.297228:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177314.297231:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177314.297233:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.297235:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177314.297237:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177314.297240:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.297241:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177314.297245:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a680.
00000020:00000040:0.0:1786177314.297247:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177314.297249:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177314.297250:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.297252:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177314.297253:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177314.297255:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177314.297259:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4000.
00000100:00000001:0.0:1786177314.297261:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177314.297266:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177314.297273:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177314.297275:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177314.297279:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000911ee1e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177314.297286:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177314.297288:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177314.297292:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177314.297293:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177314.297294:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177314.297297:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177314.297299:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7537
00000400:00000200:0.0:1786177314.297303:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177314.297306:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177314.297308:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177314.297311:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297312:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177314.297314:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297316:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177314.297318:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177314.297319:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177314.297321:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177314.297323:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177314.297325:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:0.0:1786177314.297327:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786177314.297330:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297331:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177314.297334:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177314.297336:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:2.0:1786177314.299262:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177314.299268:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177314.299271:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177314.299280:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299291:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177314.299293:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177314.299297:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177314.299300:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177314.299303:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177314.299306:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856363392:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177314.299310:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177314.299315:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177314.299318:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177314.299321:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177314.299327:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177314.299330:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177314.299337:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177314.299341:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177314.299348:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177314.299356:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856363392, portal 25
00000100:00000001:2.0:1786177314.299359:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177314.299360:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761669120 : -114547947882496 : ffff97d1ba41b800)
00000100:00000040:2.0:1786177314.299366:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299374:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177314.299377:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177314.299380:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177314.299384:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856363392, offset 0
00000400:00000200:2.0:1786177314.299390:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177314.299397:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177314.299404:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177314.299409:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177314.299413:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177314.299416:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177314.299418:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177314.299422:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255010:255010:256:4294967295] 192.168.202.110@tcp LPNI seq info [254999:254999:8:4294967295]
00000400:00000200:2.0:1786177314.299425:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177314.299435:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177314.299441:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177314.299446:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177314.299452:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177314.299456:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1786177314.299462:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177314.299465:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1786177314.299469:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177314.299472:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177314.299479:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177314.299484:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177314.299489:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177314.299491:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177314.299495:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177314.299499:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299508:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177314.299542:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299550:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177314.299552:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177314.299558:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177314.299562:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177314.299564:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302213840
00000100:00000010:2.0:1786177314.299569:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25120.
00000100:00000001:2.0:1786177314.299576:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177314.299582:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302213840
00000100:00000010:2.0:1786177314.299585:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f252a0.
00000100:00000001:2.0:1786177314.299587:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786177314.299591:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177314.299594:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177314.299596:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177314.299600:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177314/real 1786177314]  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177314.299616:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177314.299617:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177314.299619:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177314.299622:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177314.299624:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177314.299626:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177314.299629:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177314.299650:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299657:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177314.299659:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.299661:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177314.299664:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177314.299676:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299683:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177314.299688:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299694:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177314.299697:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.299698:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177314.299699:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.299701:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177314.299704:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177314.299707:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177314.299711:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177314.299713:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177314.299718:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177314.299726:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856363392:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177314.299731:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177314.299735:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299742:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177314.299743:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177314.299745:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177314.299747:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177314.299749:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177314.299752:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363520:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177314.299755:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177314.299757:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177314.299759:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177314.299761:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177314.299764:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1800.
02000000:00000001:2.0:1786177314.299766:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177314.299770:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177314.299773:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177314.299776:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177314.299779:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856363520, portal 10
00000100:00000001:2.0:1786177314.299781:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177314.299783:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761654784 : -114547947896832 : ffff97d1ba418000)
00000100:00000040:2.0:1786177314.299789:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299796:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177314.299798:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177314.299800:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177314.299803:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856363520, offset 0
00000400:00000200:2.0:1786177314.299807:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177314.299811:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177314.299815:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177314.299818:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177314.299820:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177314.299823:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177314.299825:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177314.299828:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255011:255011:256:4294967295] 192.168.202.110@tcp LPNI seq info [255000:255000:8:4294967295]
00000400:00000200:2.0:1786177314.299832:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177314.299838:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177314.299842:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177314.299845:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177314.299849:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177314.299852:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1786177314.299856:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177314.299858:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1786177314.299862:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177314.299864:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177314.299867:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177314.299870:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177314.299873:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177314.299875:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177314.299877:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177314.299881:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299888:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177314.299891:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299897:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177314.299898:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177314.299901:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177314.299905:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:2.0:1786177314.299909:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177314.299911:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302210060
00000100:00000001:2.0:1786177314.299913:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302213840
00000100:00000001:2.0:1786177314.299915:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302210060
00000100:00000001:2.0:1786177314.299918:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302213840
00000100:00000001:2.0:1786177314.299919:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177314.299921:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177314.299923:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177314.299927:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177314/real 1786177314]  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177314.299934:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177314.299935:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177314.299937:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177314.299939:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177314.299941:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177314.299943:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177314.299946:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177314.299955:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299961:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177314.299963:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.299964:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177314.299966:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177314.299969:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299976:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177314.299979:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.299985:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177314.299988:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.299989:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177314.299990:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.299992:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177314.299994:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177314.299997:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177314.300000:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177314.300002:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177314.300006:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177314.300013:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363520:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177314.300017:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177314.300021:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300027:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177314.300028:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177314.300030:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177314.300032:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177314.300034:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177314.300037:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177314.300040:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177314.300041:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177314.300043:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177314.300045:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177314.300048:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177314.300050:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177314.300053:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177314.300055:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177314.300058:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177314.300061:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856363648, portal 10
00000100:00000001:2.0:1786177314.300063:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177314.300064:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761663744 : -114547947887872 : ffff97d1ba41a300)
00000100:00000040:2.0:1786177314.300070:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300076:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177314.300078:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177314.300081:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177314.300083:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856363648, offset 0
00000400:00000200:2.0:1786177314.300087:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177314.300090:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177314.300093:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177314.300096:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177314.300098:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177314.300101:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177314.300103:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177314.300105:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255012:255012:256:4294967295] 192.168.202.110@tcp LPNI seq info [255001:255001:8:4294967295]
00000400:00000200:2.0:1786177314.300109:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177314.300115:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177314.300119:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177314.300122:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177314.300125:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177314.300128:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1786177314.300131:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177314.300134:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1786177314.300137:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177314.300139:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177314.300142:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177314.300145:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177314.300148:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177314.300149:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177314.300151:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177314.300155:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300167:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177314.300170:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300176:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177314.300177:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177314.300180:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177314.300182:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:2.0:1786177314.300185:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177314.300187:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302210185
00000100:00000001:2.0:1786177314.300190:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302213840
00000100:00000001:2.0:1786177314.300192:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302210185
00000100:00000001:2.0:1786177314.300193:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302213840
00000100:00000001:2.0:1786177314.300195:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177314.300197:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177314.300198:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177314.300202:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177314/real 1786177314]  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177314.300209:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177314.300210:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177314.300211:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177314.300214:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177314.300215:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177314.300217:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177314.300221:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177314.300228:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300234:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177314.300236:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.300237:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177314.300239:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177314.300242:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300249:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177314.300252:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300259:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177314.300261:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.300263:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177314.300264:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.300266:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177314.300268:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177314.300270:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177314.300273:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177314.300275:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177314.300279:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177314.300287:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363648:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177314.300291:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177314.300295:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300301:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177314.300303:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177314.300305:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177314.300306:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177314.300308:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177314.300312:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177314.300314:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177314.300316:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177314.300318:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177314.300320:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177314.300323:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab0800.
02000000:00000001:2.0:1786177314.300324:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177314.300327:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177314.300330:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177314.300333:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177314.300336:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856363776, portal 4
00000100:00000001:2.0:1786177314.300337:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177314.300339:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761664640 : -114547947886976 : ffff97d1ba41a680)
00000100:00000040:2.0:1786177314.300344:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300350:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177314.300351:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177314.300353:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177314.300355:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856363776, offset 0
00000400:00000200:2.0:1786177314.300359:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177314.300363:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177314.300366:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177314.300369:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177314.300371:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177314.300373:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177314.300375:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177314.300378:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255013:255013:256:4294967295] 192.168.202.110@tcp LPNI seq info [255002:255002:8:4294967295]
00000400:00000200:2.0:1786177314.300382:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177314.300388:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177314.300392:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177314.300395:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177314.300399:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177314.300401:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1786177314.300405:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177314.300408:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1786177314.300411:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177314.300413:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177314.300416:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177314.300419:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177314.300421:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177314.300423:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177314.300425:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177314.300428:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300435:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177314.300438:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300444:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177314.300445:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177314.300448:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177314.300451:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:2.0:1786177314.300454:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177314.300456:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302210060
00000100:00000001:2.0:1786177314.300458:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302213840
00000100:00000001:2.0:1786177314.300460:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302210060
00000100:00000001:2.0:1786177314.300462:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302213840
00000100:00000001:2.0:1786177314.300464:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177314.300466:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177314.300468:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177314.300472:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177314/real 1786177314]  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177314.300479:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177314.300481:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177314.300482:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177314.300484:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177314.300486:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177314.300488:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177314.300491:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177314.300498:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300503:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177314.300505:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.300506:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177314.300507:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177314.300529:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300536:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177314.300539:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300545:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177314.300548:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.300549:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177314.300551:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.300552:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177314.300554:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177314.300556:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177314.300560:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177314.300562:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177314.300566:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177314.300578:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363776:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177314.300582:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.300584:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177314.300587:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b800 x1872934856363392/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300593:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177314.300596:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177314.300598:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177314.300601:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177314.300603:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177314.300605:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1786177314.300607:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177314.300609:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7800.
02000000:00000001:2.0:1786177314.300612:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177314.300614:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177314.300615:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b800.
00000100:00000001:2.0:1786177314.300618:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177314.300620:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.300621:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177314.300624:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418000 x1872934856363520/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300642:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177314.300644:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177314.300646:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1800.
02000000:00000001:2.0:1786177314.300648:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177314.300649:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177314.300650:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177314.300652:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177314.300653:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6000.
02000000:00000001:2.0:1786177314.300656:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177314.300657:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177314.300658:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418000.
00000100:00000001:2.0:1786177314.300662:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177314.300663:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.300665:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177314.300668:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a300 x1872934856363648/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300674:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177314.300676:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177314.300677:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177314.300679:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177314.300680:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177314.300681:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177314.300683:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177314.300684:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:2.0:1786177314.300687:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177314.300688:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177314.300689:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a300.
00000100:00000001:2.0:1786177314.300691:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177314.300692:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.300694:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177314.300697:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a680 x1872934856363776/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.300703:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177314.300704:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177314.300705:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab0800.
02000000:00000001:2.0:1786177314.300708:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177314.300710:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177314.300711:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177314.300712:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177314.300714:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4000.
02000000:00000001:2.0:1786177314.300716:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177314.300717:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177314.300719:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a680.
00000100:00000001:2.0:1786177314.300721:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177314.300722:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.300725:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.300732:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177314.300734:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177314.300736:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177314.300738:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1786177314.300823:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177314.300827:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177314.300830:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177314.300833:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177314.300837:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177314.300840:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.300842:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177314.300846:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177314.300848:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.300850:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177314.300855:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419500.
00000020:00000040:0.0:1786177314.300858:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177314.300862:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177314.300864:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177314.300866:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177314.300868:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177314.300872:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177314.300876:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
00000100:00000001:0.0:1786177314.300882:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177314.300888:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177314.300896:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177314.300900:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177314.300904:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b6718a9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177314.300922:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177314.300928:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7547)
00000100:00000001:2.0:1786177314.301157:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177314.301160:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177314.301161:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177314.301166:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.301173:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177314.301175:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177314.301177:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177314.301179:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177314.301181:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177314.301184:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363904:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177314.301187:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177314.301189:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177314.301190:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177314.301192:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177314.301196:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab0800.
02000000:00000001:2.0:1786177314.301198:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177314.301201:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177314.301204:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177314.301207:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177314.301211:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856363904, portal 4
00000100:00000001:2.0:1786177314.301213:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177314.301214:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761660160 : -114547947891456 : ffff97d1ba419500)
00000100:00000040:2.0:1786177314.301220:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.301227:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177314.301229:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177314.301231:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177314.301234:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856363904, offset 0
00000400:00000200:2.0:1786177314.301238:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177314.301242:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177314.301246:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177314.301249:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177314.301251:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177314.301254:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177314.301256:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177314.301259:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255014:255014:256:4294967295] 192.168.202.110@tcp LPNI seq info [255003:255003:8:4294967295]
00000400:00000200:2.0:1786177314.301263:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177314.301270:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177314.301274:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177314.301277:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177314.301281:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177314.301284:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1786177314.301288:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177314.301291:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1786177314.301295:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177314.301296:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177314.301299:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177314.301302:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177314.301305:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177314.301307:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177314.301310:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177314.301313:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.301320:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177314.301323:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.301329:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177314.301331:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177314.301334:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177314.301336:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:2.0:1786177314.301340:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177314.301342:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302210069
00000100:00000001:2.0:1786177314.301345:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302213840
00000100:00000001:2.0:1786177314.301347:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302210069
00000100:00000001:2.0:1786177314.301349:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302213840
00000100:00000001:2.0:1786177314.301351:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177314.301353:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177314.301354:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177314.301358:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177314/real 1786177314]  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177314.301365:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177314.301366:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177314.301368:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177314.301370:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177314.301371:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177314.301374:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177314.301377:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177314.301384:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.301390:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177314.301391:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.301393:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177314.301394:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177314.301398:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.301404:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177314.301408:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.301414:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177314.301416:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.301418:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177314.301419:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.301421:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177314.301423:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177314.301425:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177314.301429:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177314.301431:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177314.301434:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177314.301441:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856363904:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177314.301445:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.301447:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177314.301449:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419500 x1872934856363904/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177314.301455:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177314.301457:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177314.301458:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab0800.
02000000:00000001:2.0:1786177314.301461:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177314.301462:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177314.301464:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177314.301466:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177314.301467:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:2.0:1786177314.301469:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177314.301471:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177314.301472:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419500.
00000100:00000001:2.0:1786177314.301474:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177314.301476:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.301477:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177314.301480:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177314.301482:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177314.301483:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177314.301485:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177314.872644:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177314.872650:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177314.872656:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177314.872657:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177314.872660:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177314.872661:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177314.872811:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177314.872815:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177314.872818:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177315.320990:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177315.321004:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177315.321006:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177315.321015:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177315.321021:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177315.321026:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177315.321030:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177315.321033:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177315.321036:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177315.896862:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177315.896872:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177315.896879:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177315.896880:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177315.896883:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177315.896885:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177315.897009:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177315.897013:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177315.897016:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177316.344691:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177316.344701:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177316.344704:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177316.344710:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177316.344715:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177316.344719:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177316.344723:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177316.344726:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177316.344729:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177316.920679:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177316.920687:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177316.920694:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177316.920696:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177316.920699:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177316.920700:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177316.920909:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177316.920913:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177316.920917:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177317.370772:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177317.370783:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177317.370786:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177317.370794:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177317.370799:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177317.370803:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177317.370806:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177317.370810:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177317.370813:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177317.944690:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177317.944699:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177317.944707:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177317.944708:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177317.944713:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177317.944718:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177317.944838:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177317.944842:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177317.944845:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177318.136704:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302213840, at 4302213888
00000100:00000040:0.0:1786177318.136707:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302213840, at 4302213888
00000100:00000010:1.0:1786177318.136722:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f252a0.
00000100:00000010:0.0:1786177318.136723:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25120.
00000100:00000001:3.0:1786177318.392666:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177318.392677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177318.392680:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177318.392687:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177318.392692:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177318.392697:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177318.392700:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177318.392704:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177318.392707:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177318.968706:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177318.968717:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177318.968724:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177318.968726:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177318.968729:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177318.968730:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177318.969574:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177318.969580:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177318.969585:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177319.416684:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177319.416696:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177319.416698:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177319.416705:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177319.416709:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177319.416714:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177319.416717:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177319.416720:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177319.416723:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177319.417185:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177319.417191:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1786177319.417193:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177319.417197:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177319.417201:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177319.417208:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7542
00000400:00000200:1.0:1786177319.417216:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177319.417222:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177319.417224:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177319.417227:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417230:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177319.417232:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417233:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177319.417237:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177319.417239:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:1.0:1786177319.417241:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177319.417244:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:1.0:1786177319.417246:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:1.0:1786177319.417247:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:1.0:1786177319.417249:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417252:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177319.417254:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417256:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177319.417259:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:1.0:1786177319.417262:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417265:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177319.417268:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177319.417272:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177319.417275:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.417277:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177319.417280:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177319.417292:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa5180.
00000020:00000040:1.0:1786177319.417297:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1786177319.417301:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177319.417303:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.417307:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177319.417311:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177319.417316:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177319.417324:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b03000.
00000100:00000001:1.0:1786177319.417332:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177319.417344:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177319.417356:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177319.417361:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177319.417366:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fbadc60a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177319.417388:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177319.417392:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177319.417408:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1786177319.417410:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177319.417412:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177319.417416:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177319.417420:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7542
00000100:00000001:2.0:1786177319.417420:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177319.417426:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000400:00000200:1.0:1786177319.417428:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177319.417431:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000400:00000200:1.0:1786177319.417433:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177319.417436:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177319.417441:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:2.0:1786177319.417442:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177319.417443:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177319.417446:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417450:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177319.417456:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:2.0:1786177319.417461:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177319.417462:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177319.417467:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:1.0:1786177319.417468:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
02000000:00000001:2.0:1786177319.417470:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:1.0:1786177319.417472:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
02000000:00000001:2.0:1786177319.417474:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1786177319.417476:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00100000:2.0:1786177319.417480:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856364032:192.168.202.110@tcp:250:kworker.0
00000020:00000001:1.0:1786177319.417481:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:1.0:1786177319.417486:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:2.0:1786177319.417488:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:1.0:1786177319.417493:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:2.0:1786177319.417493:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177319.417497:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.417498:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1786177319.417500:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:1.0:1786177319.417503:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417508:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
02000000:00000010:2.0:1786177319.417508:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177319.417537:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177319.417543:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177319.417548:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177319.417557:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177319.417566:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856364032, portal 25
00000100:00000001:2.0:1786177319.417569:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177319.417570:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751746944 : -114547957804672 : ffff97d1b9aa5180)
00000100:00000040:2.0:1786177319.417576:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.417592:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177319.417595:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177319.417599:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177319.417608:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856364032, offset 0
00000400:00000200:2.0:1786177319.417616:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177319.417626:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177319.417633:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177319.417639:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177319.417644:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177319.417652:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177319.417655:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000020:00000040:1.0:1786177319.417662:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:1.0:1786177319.417667:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417670:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177319.417674:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1786177319.417675:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255015:255015:256:4294967295] 192.168.202.110@tcp LPNI seq info [255004:255004:8:4294967295]
02000000:00000001:1.0:1786177319.417681:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1786177319.417682:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:1.0:1786177319.417687:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.417691:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177319.417696:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1786177319.417702:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177319.417702:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1786177319.417707:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177319.417713:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa4a80.
00000800:00000200:2.0:1786177319.417715:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:1.0:1786177319.417717:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786177319.417722:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000010:2.0:1786177319.417722:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
02000000:00000001:1.0:1786177319.417726:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.417729:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177319.417731:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177319.417734:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786177319.417735:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:1.0:1786177319.417741:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b01800.
00000800:00000200:2.0:1786177319.417741:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1786177319.417747:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1786177319.417752:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00100000:1.0:1786177319.417758:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1786177319.417759:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1786177319.417770:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000100:2.0:1786177319.417770:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177319.417774:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000100:00000001:1.0:1786177319.417775:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177319.417782:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005558500f] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786177319.417785:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177319.417795:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1786177319.417796:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177319.417800:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1786177319.417802:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000100:00080000:1.0:1786177319.417804:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:2.0:1786177319.417806:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:1.0:1786177319.417808:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177319.417811:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177319.417814:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1786177319.417815:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177319.417820:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7542
00000400:00000200:1.0:1786177319.417826:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177319.417830:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177319.417833:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177319.417835:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417838:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177319.417839:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417841:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177319.417844:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177319.417845:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:1.0:1786177319.417846:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177319.417849:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:1.0:1786177319.417851:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:1.0:1786177319.417853:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:1.0:1786177319.417856:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417858:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177319.417859:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417861:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177319.417863:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:1.0:1786177319.417866:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177319.417868:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177319.417870:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177319.417873:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177319.417875:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.417877:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177319.417879:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1786177319.417881:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.417883:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177319.417888:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa4e00.
00000020:00000040:1.0:1786177319.417891:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786177319.417893:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177319.417894:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.417896:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177319.417897:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177319.417899:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177319.418065:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b03c00.
00000100:00000001:1.0:1786177319.418070:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177319.418076:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177319.418085:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177319.418087:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177319.418091:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000184a4482] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177319.418099:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177319.418102:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177319.418105:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1786177319.418106:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177319.418108:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177319.418111:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177319.418113:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7542
00000400:00000200:1.0:1786177319.418118:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177319.418121:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177319.418123:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177319.418125:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.418128:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177319.418129:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.418131:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177319.418134:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177319.418135:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:1.0:1786177319.418137:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177319.418139:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:1.0:1786177319.418141:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:1.0:1786177319.418143:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:1.0:1786177319.418146:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.418148:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177319.418150:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.418151:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177319.418154:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:1.0:1786177319.418156:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177319.418159:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177319.418160:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177319.418164:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177319.418166:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.418168:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177319.418171:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1786177319.418174:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.418175:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177319.418179:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa7b80.
00000020:00000040:1.0:1786177319.418181:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:1.0:1786177319.418184:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177319.418185:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.418186:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177319.418188:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177319.418190:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177319.418194:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b00c00.
00000100:00000001:1.0:1786177319.418197:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177319.418202:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177319.418210:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177319.418212:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177319.418215:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067cf9a11] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177319.418221:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177319.418222:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177319.418225:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1786177319.418226:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177319.418227:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177319.418229:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177319.418231:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7542
00000400:00000200:1.0:1786177319.418233:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177319.418235:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177319.418237:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177319.418239:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.418240:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177319.418242:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.418243:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177319.418245:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177319.418246:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:1.0:1786177319.418247:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177319.418249:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:1.0:1786177319.418251:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:1.0:1786177319.418252:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:1.0:1786177319.418255:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.418257:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177319.418259:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177319.418260:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177319.418263:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:1.0:1786177319.418265:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177319.418267:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177319.418269:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177319.418271:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177319.418273:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.418275:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177319.418277:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1786177319.418280:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.418281:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177319.418285:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa6d80.
00000020:00000040:1.0:1786177319.418287:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:1.0:1786177319.418289:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177319.418290:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177319.418292:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177319.418294:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177319.418295:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177319.418299:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b02c00.
00000100:00000001:1.0:1786177319.418303:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177319.418307:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177319.418318:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177319.418320:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177319.418323:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038d1214c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177319.418331:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177319.418336:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7552)
00000100:00000200:2.0:1786177319.418626:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.418637:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177319.418640:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.418646:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177319.418647:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177319.418655:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177319.418658:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177319.418662:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302218959
00000100:00000010:2.0:1786177319.418667:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25e40.
00000100:00000001:2.0:1786177319.418675:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177319.418682:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302218959
00000100:00000010:2.0:1786177319.418684:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25f00.
00000100:00000001:2.0:1786177319.418688:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786177319.418691:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177319.418695:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177319.418698:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177319.418703:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177319/real 1786177319]  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177319.418715:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177319.418718:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177319.418720:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177319.418723:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177319.418725:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177319.418727:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177319.418731:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177319.418738:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.418745:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177319.418747:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.418750:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177319.418752:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177319.418757:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.418764:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177319.418769:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.418782:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177319.418785:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.418787:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177319.418789:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.418791:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177319.418794:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177319.418797:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177319.418800:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177319.418803:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177319.418808:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177319.418817:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856364032:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177319.418831:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.418835:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177319.418838:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5180 x1872934856364032/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.418845:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177319.418848:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177319.418851:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177319.418854:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177319.418856:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177319.418858:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1786177319.418860:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177319.418863:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03000.
02000000:00000001:2.0:1786177319.418868:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177319.418870:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177319.418872:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5180.
00000100:00000001:2.0:1786177319.418875:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177319.418877:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.418879:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.418888:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177319.418891:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177319.418893:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177319.418895:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177319.418896:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177319.418900:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.418906:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177319.418907:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177319.418909:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177319.418911:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177319.418914:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177319.418917:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856364160:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177319.418920:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177319.418923:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177319.418925:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177319.418927:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177319.418930:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177319.418932:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177319.418936:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177319.418939:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177319.418943:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177319.418948:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856364160, portal 10
00000100:00000001:2.0:1786177319.418950:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177319.418952:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751745152 : -114547957806464 : ffff97d1b9aa4a80)
00000100:00000040:2.0:1786177319.418959:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.418965:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177319.418967:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177319.418969:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177319.418972:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856364160, offset 0
00000400:00000200:2.0:1786177319.418976:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177319.418981:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177319.418987:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177319.418990:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177319.418992:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177319.418995:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177319.418998:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177319.419001:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255016:255016:256:4294967295] 192.168.202.110@tcp LPNI seq info [255005:255005:8:4294967295]
00000400:00000200:2.0:1786177319.419005:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177319.419013:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177319.419019:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177319.419023:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177319.419028:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177319.419031:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1786177319.419036:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177319.419039:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1786177319.419042:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177319.419044:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177319.419049:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177319.419052:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177319.419056:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177319.419058:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177319.419060:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177319.419064:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419072:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177319.419075:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419082:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177319.419083:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177319.419086:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177319.419090:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:2.0:1786177319.419105:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177319.419107:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302215179
00000100:00000001:2.0:1786177319.419111:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302218959
00000100:00000001:2.0:1786177319.419113:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302215179
00000100:00000001:2.0:1786177319.419115:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302218959
00000100:00000001:2.0:1786177319.419117:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177319.419119:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177319.419121:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177319.419125:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177319/real 1786177319]  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177319.419132:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177319.419134:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177319.419135:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177319.419138:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177319.419139:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177319.419141:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177319.419144:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177319.419152:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419159:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177319.419160:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419162:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177319.419163:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177319.419167:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419173:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177319.419175:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419180:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177319.419182:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419183:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177319.419184:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419185:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177319.419187:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177319.419188:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177319.419190:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177319.419191:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177319.419193:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177319.419199:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856364160:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177319.419201:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177319.419204:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419207:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177319.419208:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177319.419209:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177319.419210:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177319.419212:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177319.419213:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856364288:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177319.419215:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177319.419217:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177319.419218:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177319.419219:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177319.419223:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1800.
02000000:00000001:2.0:1786177319.419225:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177319.419228:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177319.419231:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177319.419234:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177319.419237:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856364288, portal 10
00000100:00000001:2.0:1786177319.419239:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177319.419240:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751746048 : -114547957805568 : ffff97d1b9aa4e00)
00000100:00000040:2.0:1786177319.419247:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419253:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177319.419256:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177319.419258:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177319.419260:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856364288, offset 0
00000400:00000200:2.0:1786177319.419264:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177319.419268:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177319.419271:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177319.419274:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177319.419276:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177319.419277:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177319.419279:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177319.419281:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255017:255017:256:4294967295] 192.168.202.110@tcp LPNI seq info [255006:255006:8:4294967295]
00000400:00000200:2.0:1786177319.419283:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177319.419288:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177319.419290:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177319.419292:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177319.419294:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177319.419297:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1786177319.419299:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177319.419301:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1786177319.419303:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177319.419305:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177319.419307:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177319.419309:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177319.419311:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177319.419312:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177319.419314:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177319.419316:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419325:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177319.419327:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419331:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177319.419331:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177319.419333:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177319.419335:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:2.0:1786177319.419337:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177319.419338:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302215304
00000100:00000001:2.0:1786177319.419340:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302218959
00000100:00000001:2.0:1786177319.419341:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302215304
00000100:00000001:2.0:1786177319.419343:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302218959
00000100:00000001:2.0:1786177319.419344:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177319.419346:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177319.419347:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177319.419349:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177319/real 1786177319]  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177319.419354:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177319.419355:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177319.419357:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177319.419358:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177319.419360:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177319.419361:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177319.419364:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177319.419372:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419378:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177319.419380:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419381:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177319.419382:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177319.419386:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419392:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177319.419396:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419402:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177319.419403:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419404:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177319.419405:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419406:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177319.419407:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177319.419409:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177319.419411:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177319.419412:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177319.419415:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177319.419420:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856364288:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177319.419422:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177319.419424:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419427:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177319.419428:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177319.419430:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177319.419431:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177319.419432:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177319.419434:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856364416:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177319.419436:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177319.419437:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177319.419439:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177319.419440:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177319.419442:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177319.419443:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177319.419445:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177319.419447:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177319.419449:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177319.419451:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856364416, portal 4
00000100:00000001:2.0:1786177319.419452:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177319.419453:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751757696 : -114547957793920 : ffff97d1b9aa7b80)
00000100:00000040:2.0:1786177319.419458:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419461:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177319.419463:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177319.419464:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177319.419465:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856364416, offset 0
00000400:00000200:2.0:1786177319.419467:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177319.419470:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177319.419472:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177319.419473:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177319.419475:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177319.419476:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177319.419478:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177319.419480:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255018:255018:256:4294967295] 192.168.202.110@tcp LPNI seq info [255007:255007:8:4294967295]
00000400:00000200:2.0:1786177319.419482:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177319.419486:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177319.419489:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177319.419491:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177319.419494:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177319.419496:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1786177319.419498:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177319.419500:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1786177319.419503:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177319.419504:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177319.419506:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177319.419508:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177319.419531:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177319.419533:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177319.419536:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177319.419540:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419547:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177319.419550:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419557:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177319.419558:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177319.419561:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177319.419564:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:2.0:1786177319.419567:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177319.419569:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302215180
00000100:00000001:2.0:1786177319.419571:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302218959
00000100:00000001:2.0:1786177319.419573:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302215180
00000100:00000001:2.0:1786177319.419576:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302218959
00000100:00000001:2.0:1786177319.419578:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177319.419580:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177319.419591:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177319.419595:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177319/real 1786177319]  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177319.419604:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177319.419605:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177319.419607:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177319.419609:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177319.419611:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177319.419617:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177319.419620:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177319.419627:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419633:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177319.419634:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419636:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177319.419638:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177319.419641:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419647:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177319.419650:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419657:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177319.419659:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419661:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177319.419662:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419663:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177319.419665:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177319.419667:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177319.419670:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177319.419672:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177319.419676:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177319.419693:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856364416:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177319.419697:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177319.419699:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419705:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177319.419707:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177319.419709:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177319.419711:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177319.419712:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177319.419715:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856364544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177319.419717:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177319.419719:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177319.419721:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177319.419722:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177319.419726:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177319.419728:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177319.419732:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177319.419734:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177319.419737:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177319.419740:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856364544, portal 4
00000100:00000001:2.0:1786177319.419746:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177319.419748:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751754112 : -114547957797504 : ffff97d1b9aa6d80)
00000100:00000040:2.0:1786177319.419754:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419760:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177319.419762:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177319.419765:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177319.419768:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856364544, offset 0
00000400:00000200:2.0:1786177319.419772:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177319.419776:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177319.419779:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177319.419782:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177319.419784:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177319.419786:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177319.419791:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177319.419793:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255019:255019:256:4294967295] 192.168.202.110@tcp LPNI seq info [255008:255008:8:4294967295]
00000400:00000200:2.0:1786177319.419797:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177319.419803:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177319.419807:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177319.419811:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177319.419814:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177319.419817:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1786177319.419821:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177319.419824:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1786177319.419827:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177319.419828:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177319.419831:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177319.419835:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177319.419838:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177319.419839:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177319.419841:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177319.419845:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419852:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177319.419855:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419861:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177319.419862:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177319.419865:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177319.419868:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:2.0:1786177319.419871:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177319.419874:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302215188
00000100:00000001:2.0:1786177319.419876:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302218959
00000100:00000001:2.0:1786177319.419878:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302215188
00000100:00000001:2.0:1786177319.419881:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302218959
00000100:00000001:2.0:1786177319.419883:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177319.419885:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177319.419887:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177319.419891:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177319/real 1786177319]  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177319.419898:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177319.419899:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177319.419901:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177319.419903:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177319.419905:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177319.419907:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177319.419909:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177319.419915:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419922:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177319.419923:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419925:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177319.419926:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177319.419929:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419936:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177319.419939:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419946:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177319.419950:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419951:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177319.419952:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419954:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177319.419955:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177319.419958:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177319.419961:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177319.419963:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177319.419967:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177319.419975:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856364544:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177319.419979:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.419981:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177319.419984:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4a80 x1872934856364160/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.419991:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177319.419993:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177319.419995:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177319.419997:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177319.419999:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177319.420001:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177319.420002:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177319.420004:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b01800.
02000000:00000001:2.0:1786177319.420009:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177319.420010:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177319.420012:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4a80.
00000100:00000001:2.0:1786177319.420015:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177319.420016:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.420018:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177319.420021:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4e00 x1872934856364288/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.420033:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177319.420035:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177319.420036:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1800.
02000000:00000001:2.0:1786177319.420039:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177319.420040:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177319.420041:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177319.420043:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177319.420045:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03c00.
02000000:00000001:2.0:1786177319.420047:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177319.420048:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177319.420050:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4e00.
00000100:00000001:2.0:1786177319.420052:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177319.420053:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.420055:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177319.420058:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa7b80 x1872934856364416/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.420064:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177319.420065:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177319.420066:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177319.420069:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177319.420070:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177319.420071:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177319.420073:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177319.420074:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00c00.
02000000:00000001:2.0:1786177319.420076:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177319.420078:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177319.420079:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa7b80.
00000100:00000001:2.0:1786177319.420081:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177319.420083:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.420084:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177319.420088:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa6d80 x1872934856364544/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177319.420099:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177319.420100:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177319.420102:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177319.420104:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177319.420106:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177319.420107:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177319.420108:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177319.420109:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b02c00.
02000000:00000001:2.0:1786177319.420111:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177319.420112:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177319.420113:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa6d80.
00000100:00000001:2.0:1786177319.420116:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177319.420117:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.420119:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177319.420124:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177319.420125:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177319.420127:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177319.420129:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177319.992710:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177319.992720:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177319.992726:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177319.992728:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177319.992732:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177319.992733:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177319.992803:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177319.992808:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177319.992812:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177320.440668:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177320.440686:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177320.440689:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177320.440696:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177320.440701:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177320.440704:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177320.440707:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177320.440710:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177320.440714:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177321.016646:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177321.016653:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177321.016658:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177321.016658:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177321.016661:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177321.016662:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177321.016731:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177321.016736:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177321.016740:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177321.464697:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177321.464708:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177321.464710:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177321.464717:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177321.464721:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177321.464726:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177321.464730:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177321.464733:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177321.464736:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177322.042732:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177322.042743:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177322.042746:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177322.042750:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177322.042756:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177322.042759:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177322.042761:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177322.042764:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177322.042768:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177322.042770:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177322.042771:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177322.042773:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177322.042775:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177322.042776:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177322.042778:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177322.042779:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177322.042783:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177322.042786:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177322.042855:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177322.042859:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177322.042862:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177322.488689:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177322.488698:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177322.488700:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177322.488705:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177322.488709:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177322.488712:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177322.488715:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177322.488718:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177322.488726:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177323.064660:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177323.064666:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177323.064671:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177323.064672:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177323.064674:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177323.064675:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177323.064761:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177323.064768:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177323.064773:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177323.256640:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302218959, at 4302219008
00000100:00000040:1.0:1786177323.256640:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302218959, at 4302219008
00000100:00000010:1.0:1786177323.256661:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25f00.
00000100:00000010:0.0:1786177323.256663:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25e40.
00000100:00000001:3.0:1786177323.512632:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177323.512638:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177323.512639:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177323.512643:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177323.512646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177323.512649:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177323.512651:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177323.512653:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177323.512655:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177324.088650:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177324.088657:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177324.088662:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177324.088663:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177324.088667:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177324.088670:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177324.088672:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177324.088675:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177324.088678:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177324.088680:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177324.088682:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177324.088684:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177324.088830:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177324.088834:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177324.088837:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177324.536646:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177324.536654:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177324.536656:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177324.536667:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177324.536670:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177324.536674:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177324.536676:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177324.536678:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177324.536681:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177324.536690:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177324.536701:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177324.536704:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177324.536710:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177324.536714:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177324.536721:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7547
00000400:00000200:0.0:1786177324.536731:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177324.536738:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177324.536741:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177324.536745:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.536749:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177324.536751:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.536752:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177324.536756:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177324.536758:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177324.536760:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177324.536763:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177324.536765:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177324.536767:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177324.536769:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.536770:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177324.536771:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.536772:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177324.536774:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177324.536776:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177324.536777:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177324.536780:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177324.536783:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177324.536785:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177324.536787:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177324.536788:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177324.536799:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41aa00.
00000020:00000040:0.0:1786177324.536802:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177324.536806:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177324.536807:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177324.536809:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177324.536812:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177324.536816:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177324.536823:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7400.
00000100:00000001:0.0:1786177324.536830:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177324.536839:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177324.536848:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177324.536852:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177324.536855:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da0255a4] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177324.536877:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177324.536880:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177324.536887:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177324.536889:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177324.536890:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177324.536893:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177324.536896:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7547
00000400:00000200:0.0:1786177324.536899:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177324.536902:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177324.536904:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177324.536907:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.536909:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177324.536911:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.536913:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177324.536915:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177324.536917:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177324.536918:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177324.536920:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:2.0:1786177324.536920:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1786177324.536922:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177324.536924:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:2.0:1786177324.536925:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1786177324.536927:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177324.536928:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1786177324.536929:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177324.536931:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.536932:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177324.536935:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177324.536938:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177324.536940:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177324.536942:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1786177324.536943:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177324.536947:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177324.536950:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177324.536951:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177324.536952:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177324.536953:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1786177324.536956:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1786177324.536957:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:0.0:1786177324.536959:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177324.536960:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1786177324.536961:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177324.536963:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177324.536965:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41ad80.
00000100:00100000:2.0:1786177324.536966:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856364672:192.168.202.110@tcp:250:kworker.0
00000020:00000040:0.0:1786177324.536967:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177324.536969:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177324.536970:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.536970:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1786177324.536972:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177324.536973:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177324.536975:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177324.536976:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000010:0.0:1786177324.536979:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
02000000:00000001:2.0:1786177324.536979:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177324.536982:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177324.536982:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:0.0:1786177324.536986:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:2.0:1786177324.536987:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177324.536989:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177324.536993:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177324.536995:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177324.536998:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034e2c676] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786177324.537000:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177324.537004:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000100:00000001:0.0:1786177324.537006:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177324.537008:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177324.537012:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177324.537013:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:2.0:1786177324.537013:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00080000:0.0:1786177324.537015:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177324.537018:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177324.537019:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7547
00000100:00000200:2.0:1786177324.537022:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856364672, portal 25
00000400:00000200:0.0:1786177324.537023:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177324.537025:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000400:00000200:0.0:1786177324.537026:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1786177324.537026:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761665536 : -114547947886080 : ffff97d1ba41aa00)
00000100:00080000:0.0:1786177324.537028:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177324.537031:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:2.0:1786177324.537032:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177324.537033:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177324.537035:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.537037:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177324.537039:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177324.537041:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177324.537042:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177324.537044:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177324.537046:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177324.537048:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000001:2.0:1786177324.537050:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1786177324.537051:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.537053:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177324.537053:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000040:0.0:1786177324.537055:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000010:2.0:1786177324.537056:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000001:0.0:1786177324.537057:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177324.537059:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00000200:2.0:1786177324.537060:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856364672, offset 0
00000100:00080000:0.0:1786177324.537061:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177324.537064:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177324.537065:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177324.537067:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1786177324.537067:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786177324.537069:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177324.537071:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177324.537073:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177324.537075:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177324.537076:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177324.537077:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177324.537080:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b480.
00000020:00000040:0.0:1786177324.537081:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000400:00000200:2.0:1786177324.537083:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
02000000:00000001:0.0:1786177324.537084:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177324.537085:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177324.537086:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177324.537088:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177324.537089:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177324.537089:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000010:0.0:1786177324.537093:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000400:00000200:2.0:1786177324.537093:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1786177324.537096:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177324.537098:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00100000:0.0:1786177324.537099:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1786177324.537100:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177324.537105:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255020:255020:256:4294967295] 192.168.202.110@tcp LPNI seq info [255009:255009:8:4294967295]
00000100:00000001:0.0:1786177324.537106:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177324.537108:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177324.537109:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177324.537112:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001215e9e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177324.537118:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177324.537120:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1786177324.537121:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177324.537124:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177324.537126:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:2.0:1786177324.537126:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177324.537127:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177324.537130:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177324.537132:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7547
00000800:00000010:2.0:1786177324.537132:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000400:00000200:0.0:1786177324.537135:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177324.537137:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177324.537139:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000800:00000200:2.0:1786177324.537139:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:0.0:1786177324.537142:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.537144:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1786177324.537144:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:0.0:1786177324.537146:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.537147:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177324.537149:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000800:00000200:2.1:1786177324.537150:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:0.0:1786177324.537151:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177324.537153:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000800:00000200:2.1:1786177324.537154:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000020:00000040:0.0:1786177324.537155:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177324.537157:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177324.537159:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000800:00000100:2.0:1786177324.537159:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:0.0:1786177324.537161:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000010:2.0:1786177324.537161:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000100:00000001:0.0:1786177324.537163:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177324.537165:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.537166:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177324.537168:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177324.537170:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1786177324.537171:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1786177324.537173:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177324.537174:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1786177324.537176:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1786177324.537177:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177324.537178:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177324.537180:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177324.537182:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000200:2.0:1786177324.537182:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000008:00000001:0.0:1786177324.537184:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1786177324.537185:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:0.0:1786177324.537186:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177324.537188:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b100.
00000100:00000001:2.0:1786177324.537189:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:0.0:1786177324.537190:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177324.537192:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:2.0:1786177324.537193:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177324.537194:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177324.537195:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177324.537197:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177324.537198:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177324.537201:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:2.0:1786177324.537202:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177324.537204:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177324.537207:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177324.537214:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177324.537216:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177324.537216:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177324.537221:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000fa632fb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.537225:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.537227:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1786177324.537228:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177324.537230:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177324.537234:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177324.537234:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177324.537235:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177324.537237:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1786177324.537238:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177324.537239:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177324.537241:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7547
00000100:00000001:2.0:1786177324.537241:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302224077
00000400:00000200:0.0:1786177324.537244:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177324.537247:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000010:2.0:1786177324.537247:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25240.
00000100:00080000:0.0:1786177324.537249:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177324.537251:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.537253:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177324.537254:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177324.537255:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:0.0:1786177324.537257:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177324.537258:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177324.537260:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177324.537261:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:2.0:1786177324.537262:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302224077
00000020:00000040:0.0:1786177324.537264:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00000010:2.0:1786177324.537265:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25ba0.
00000020:00000001:0.0:1786177324.537266:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:0.0:1786177324.537268:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:2.0:1786177324.537269:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:0.0:1786177324.537270:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177324.537272:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1786177324.537272:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1786177324.537274:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177324.537275:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177324.537276:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177324.537278:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000001:2.0:1786177324.537278:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00080000:0.0:1786177324.537280:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177324.537282:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177324.537282:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177324/real 1786177324]  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1786177324.537284:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177324.537287:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177324.537289:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177324.537291:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000001:2.0:1786177324.537292:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000008:00100000:0.0:1786177324.537293:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000400:00000001:2.0:1786177324.537293:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000008:00000001:0.0:1786177324.537295:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1786177324.537296:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000100:00000001:0.0:1786177324.537297:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177324.537298:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000100:00000010:0.0:1786177324.537299:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419180.
00000400:00000010:2.0:1786177324.537300:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000020:00000040:0.0:1786177324.537302:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:2.0:1786177324.537303:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:0.0:1786177324.537304:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177324.537305:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1786177324.537306:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177324.537307:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177324.537309:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177324.537310:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177324.537314:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4c00.
00000100:00000200:2.0:1786177324.537315:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177324.537317:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177324.537321:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177324.537323:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177324.537324:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177324.537327:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177324.537327:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177324.537329:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.537329:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177324.537333:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005a89d8ab] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177324.537334:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177324.537342:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.537342:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177324.537346:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1786177324.537347:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7557)
00000100:00000001:2.0:1786177324.537363:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177324.537366:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.537367:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177324.537368:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.537371:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177324.537373:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177324.537376:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177324.537380:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177324.537383:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177324.537388:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177324.537401:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856364672:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177324.537406:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.537408:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177324.537411:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41aa00 x1872934856364672/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.537418:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177324.537421:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177324.537424:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177324.537427:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177324.537429:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177324.537431:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1786177324.537434:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177324.537437:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7400.
02000000:00000001:2.0:1786177324.537440:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177324.537442:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177324.537453:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41aa00.
00000100:00000001:2.0:1786177324.537457:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177324.537458:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.537460:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.537469:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177324.537471:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.537473:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177324.537475:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177324.537476:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177324.537479:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.537486:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177324.537487:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177324.537489:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177324.537491:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177324.537493:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177324.537496:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856364800:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177324.537498:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177324.537500:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177324.537501:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177324.537502:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177324.537505:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177324.537507:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177324.537558:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177324.537562:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177324.537565:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177324.537568:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856364800, portal 10
00000100:00000001:2.0:1786177324.537579:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177324.537581:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761666432 : -114547947885184 : ffff97d1ba41ad80)
00000100:00000040:2.0:1786177324.537588:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.537594:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177324.537596:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177324.537599:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177324.537601:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856364800, offset 0
00000400:00000200:2.0:1786177324.537604:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177324.537609:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177324.537612:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177324.537615:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177324.537617:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177324.537620:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177324.537622:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177324.537626:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255021:255021:256:4294967295] 192.168.202.110@tcp LPNI seq info [255010:255010:8:4294967295]
00000400:00000200:2.0:1786177324.537629:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177324.537635:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177324.537639:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177324.537642:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177324.537646:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177324.537649:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1786177324.537652:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177324.537664:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1786177324.537668:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177324.537669:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177324.537672:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177324.537676:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177324.537678:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177324.537680:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177324.537682:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177324.537686:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.537699:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177324.537702:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.537707:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.537709:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177324.537712:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177324.537716:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:2.0:1786177324.537719:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177324.537721:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302220298
00000100:00000001:2.0:1786177324.537723:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302224077
00000100:00000001:2.0:1786177324.537726:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302220298
00000100:00000001:2.0:1786177324.537728:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302224077
00000100:00000001:2.0:1786177324.537730:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177324.537732:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.537733:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177324.539362:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177324/real 1786177324]  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177324.539380:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177324.539382:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177324.539385:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177324.539488:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177324.539491:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177324.539496:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177324.539500:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177324.539523:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.539541:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177324.539545:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.539548:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177324.539551:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177324.539555:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.539670:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177324.539678:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.539686:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177324.539689:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.539691:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177324.539692:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.539695:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177324.539698:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177324.539702:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177324.539706:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177324.539709:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177324.539713:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177324.539722:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856364800:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177324.539728:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177324.539732:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.539739:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177324.539742:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177324.539746:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177324.539749:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177324.539752:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177324.539756:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856364928:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177324.539760:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177324.539764:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177324.539767:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177324.539769:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177324.539779:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177324.539781:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177324.539787:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177324.539791:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177324.539796:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177324.539802:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856364928, portal 10
00000100:00000001:2.0:1786177324.539805:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177324.539807:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761668224 : -114547947883392 : ffff97d1ba41b480)
00000100:00000040:2.0:1786177324.539814:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.539821:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177324.539824:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177324.539826:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177324.539830:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856364928, offset 0
00000400:00000200:2.0:1786177324.539838:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177324.539848:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177324.539856:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177324.539862:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177324.539866:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177324.539870:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177324.539874:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177324.539878:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255022:255022:256:4294967295] 192.168.202.110@tcp LPNI seq info [255011:255011:8:4294967295]
00000400:00000200:2.0:1786177324.539882:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177324.539892:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177324.539898:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177324.539903:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177324.539910:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177324.539914:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1786177324.539920:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177324.539924:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1786177324.539929:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177324.539932:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177324.539940:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177324.539945:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177324.539950:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177324.539952:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177324.539955:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177324.539958:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.539968:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177324.539971:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.539978:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.539979:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177324.539986:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177324.539991:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:2.0:1786177324.539995:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177324.539998:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302220425
00000100:00000001:2.0:1786177324.540002:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302224077
00000100:00000001:2.0:1786177324.540004:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302220425
00000100:00000001:2.0:1786177324.540007:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302224077
00000100:00000001:2.0:1786177324.540009:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177324.540012:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.540015:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177324.540019:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177324/real 1786177324]  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177324.540027:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177324.540029:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177324.540031:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177324.540033:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177324.540034:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177324.540037:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177324.540040:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177324.540047:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540053:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177324.540054:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540056:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177324.540057:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177324.540060:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540066:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177324.540070:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540077:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177324.540079:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540081:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177324.540082:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540084:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177324.540087:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177324.540089:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177324.540093:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177324.540095:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177324.540099:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177324.540113:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856364928:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177324.540118:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177324.540121:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540128:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177324.540130:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177324.540133:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177324.540134:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177324.540136:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177324.540139:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856365056:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177324.540143:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177324.540145:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177324.540146:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177324.540148:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177324.540152:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1800.
02000000:00000001:2.0:1786177324.540154:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177324.540157:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177324.540160:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177324.540163:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177324.540166:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856365056, portal 4
00000100:00000001:2.0:1786177324.540168:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177324.540169:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761667328 : -114547947884288 : ffff97d1ba41b100)
00000100:00000040:2.0:1786177324.540175:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540180:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177324.540182:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177324.540185:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177324.540187:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856365056, offset 0
00000400:00000200:2.0:1786177324.540191:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177324.540196:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177324.540200:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177324.540202:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177324.540205:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177324.540207:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177324.540209:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177324.540212:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255023:255023:256:4294967295] 192.168.202.110@tcp LPNI seq info [255012:255012:8:4294967295]
00000400:00000200:2.0:1786177324.540216:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177324.540221:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177324.540226:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177324.540229:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177324.540233:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177324.540236:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1786177324.540240:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177324.540242:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1786177324.540246:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177324.540248:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177324.540250:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177324.540254:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177324.540256:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177324.540258:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177324.540260:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177324.540263:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540271:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177324.540274:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540281:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.540282:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177324.540285:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177324.540288:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:2.0:1786177324.540291:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177324.540294:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302220300
00000100:00000001:2.0:1786177324.540296:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302224077
00000100:00000001:2.0:1786177324.540299:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302220300
00000100:00000001:2.0:1786177324.540300:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302224077
00000100:00000001:2.0:1786177324.540302:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177324.540304:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.540306:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177324.540310:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177324/real 1786177324]  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177324.540318:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177324.540319:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177324.540321:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177324.540323:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177324.540325:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177324.540327:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177324.540330:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177324.540338:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540344:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177324.540345:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540347:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177324.540348:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177324.540351:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540390:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177324.540394:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540400:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177324.540403:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540405:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177324.540406:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540408:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177324.540410:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177324.540413:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177324.540417:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177324.540421:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177324.540426:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177324.540435:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856365056:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177324.540440:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177324.540458:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540465:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177324.540467:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177324.540469:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177324.540471:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177324.540473:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177324.540476:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856365184:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177324.540479:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177324.540482:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177324.540484:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177324.540485:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177324.540489:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177324.540490:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177324.540493:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177324.540496:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177324.540499:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177324.540503:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856365184, portal 4
00000100:00000001:2.0:1786177324.540506:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177324.540508:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761659264 : -114547947892352 : ffff97d1ba419180)
00000100:00000040:2.0:1786177324.540558:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540565:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177324.540568:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177324.540578:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177324.540581:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856365184, offset 0
00000400:00000200:2.0:1786177324.540585:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177324.540589:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177324.540593:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177324.540596:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177324.540598:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177324.540601:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177324.540603:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177324.540606:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255024:255024:256:4294967295] 192.168.202.110@tcp LPNI seq info [255013:255013:8:4294967295]
00000400:00000200:2.0:1786177324.540610:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177324.540617:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177324.540621:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177324.540625:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177324.540629:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177324.540632:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1786177324.540636:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177324.540639:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1786177324.540643:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177324.540645:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177324.540649:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177324.540653:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177324.540656:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177324.540658:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177324.540660:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177324.540664:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540678:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177324.540682:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540689:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.540691:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177324.540694:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177324.540697:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:2.0:1786177324.540701:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177324.540704:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302220309
00000100:00000001:2.0:1786177324.540706:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302224077
00000100:00000001:2.0:1786177324.540709:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302220309
00000100:00000001:2.0:1786177324.540711:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302224077
00000100:00000001:2.0:1786177324.540714:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177324.540716:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.540717:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177324.540721:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177324/real 1786177324]  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177324.540729:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177324.540731:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177324.540732:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177324.540735:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177324.540737:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177324.540739:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177324.540742:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177324.540751:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540758:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177324.540759:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540761:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177324.540762:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177324.540766:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540772:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177324.540776:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540782:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177324.540785:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540787:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177324.540788:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540790:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177324.540792:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177324.540795:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177324.540798:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177324.540800:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177324.540804:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177324.540812:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856365184:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177324.540816:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540819:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177324.540822:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41ad80 x1872934856364800/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540830:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177324.540833:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177324.540836:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177324.540839:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177324.540842:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177324.540845:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177324.540847:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177324.540850:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:2.0:1786177324.540855:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177324.540857:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177324.540859:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41ad80.
00000100:00000001:2.0:1786177324.540864:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177324.540865:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540867:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177324.540871:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b480 x1872934856364928/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540878:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177324.540879:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177324.540881:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177324.540883:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177324.540884:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177324.540886:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177324.540888:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177324.540889:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:2.0:1786177324.540892:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177324.540893:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177324.540894:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b480.
00000100:00000001:2.0:1786177324.540896:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177324.540897:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540899:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177324.540902:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b100 x1872934856365056/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540908:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177324.540910:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177324.540911:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1800.
02000000:00000001:2.0:1786177324.540913:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177324.540914:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177324.540915:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177324.540917:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177324.540919:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:2.0:1786177324.540921:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177324.540923:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177324.540924:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b100.
00000100:00000001:2.0:1786177324.540926:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177324.540927:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540929:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177324.540932:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419180 x1872934856365184/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177324.540938:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177324.540940:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177324.540941:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177324.540944:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177324.540945:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177324.540947:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177324.540949:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177324.540950:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:2.0:1786177324.540952:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177324.540954:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177324.540955:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419180.
00000100:00000001:2.0:1786177324.540958:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177324.540959:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540963:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177324.540974:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177324.540976:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177324.540979:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177324.540981:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177325.112637:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177325.112643:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177325.112649:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177325.112650:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177325.112653:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177325.112654:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177325.113050:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177325.113057:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177325.113063:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177325.560614:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177325.560620:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177325.560622:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177325.560626:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177325.560628:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177325.560631:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177325.560633:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177325.560635:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177325.560637:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177326.136642:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177326.136649:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177326.136655:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177326.136656:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177326.136659:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177326.136660:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177326.136708:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177326.136712:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177326.136715:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177326.584617:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177326.584625:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177326.584627:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177326.584632:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177326.584635:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177326.584637:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177326.584639:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177326.584641:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177326.584643:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177327.160648:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177327.160656:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177327.160662:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177327.160664:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177327.160667:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177327.160669:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177327.160841:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177327.160845:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177327.160848:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177327.608601:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177327.608609:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177327.608611:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177327.608617:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177327.608620:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177327.608624:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177327.608627:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177327.608630:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177327.608633:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177328.184613:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177328.184618:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177328.184626:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177328.184627:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177328.184630:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177328.184630:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177328.184680:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177328.184683:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177328.184686:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177328.376601:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302224077, at 4302224128
00000100:00000040:1.0:1786177328.376601:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302224077, at 4302224128
00000100:00000010:0.0:1786177328.376614:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25240.
00000100:00000010:1.0:1786177328.376616:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25ba0.
00000100:00000001:3.0:1786177328.635619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177328.635627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177328.635629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177328.635634:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177328.635638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177328.635641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177328.635645:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177328.635648:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177328.635651:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177329.208623:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177329.208627:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177329.208631:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177329.208632:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177329.208634:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177329.208635:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177329.208686:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177329.208689:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177329.208692:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177329.656573:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177329.656578:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177329.656580:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177329.656584:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177329.656588:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1786177329.656590:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177329.656591:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177329.656595:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00080000:1.0:1786177329.656596:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177329.656597:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177329.656599:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177329.656601:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177329.656603:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177329.656607:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177329.656612:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7552
00000400:00000200:1.0:1786177329.656620:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177329.656625:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177329.656627:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177329.656629:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.656631:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177329.656633:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.656634:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177329.656637:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177329.656638:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:1.0:1786177329.656644:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177329.656646:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:1.0:1786177329.656648:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:1.0:1786177329.656650:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:1.0:1786177329.656651:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.656653:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177329.656654:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.656655:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177329.656657:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:1.0:1786177329.656658:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177329.656660:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177329.656662:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177329.656664:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177329.656666:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177329.656667:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177329.656668:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177329.656675:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa4380.
00000020:00000040:1.0:1786177329.656677:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1786177329.656680:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177329.656681:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177329.656683:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177329.656689:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177329.656692:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177329.656697:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b00000.
00000100:00000001:1.0:1786177329.656702:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177329.656709:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177329.656716:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177329.656719:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177329.656723:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abaf0d59] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177329.656747:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177329.656750:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177329.656754:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1786177329.656755:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177329.656757:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177329.656760:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177329.656762:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7552
00000400:00000200:1.0:1786177329.656766:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177329.656770:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177329.656773:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177329.656776:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.656778:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1786177329.656780:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1786177329.656781:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.656785:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177329.656788:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177329.656789:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:1.0:1786177329.656791:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177329.656793:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:2.0:1786177329.656793:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177329.656797:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1786177329.656798:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:1.0:1786177329.656802:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177329.656807:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786177329.656808:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.656813:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177329.656816:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.656820:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:2.0:1786177329.656822:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000020:00000040:1.0:1786177329.656824:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00000001:2.0:1786177329.656825:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177329.656828:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
02000000:00000001:2.0:1786177329.656830:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1786177329.656831:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177329.656833:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177329.656833:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177329.656837:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177329.656840:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1786177329.656841:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856365312:192.168.202.110@tcp:250:kworker.0
02000000:00000001:1.0:1786177329.656844:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.656846:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1786177329.656847:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:2.0:1786177329.656850:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:1.0:1786177329.656851:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1786177329.656853:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1786177329.656855:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177329.656856:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1786177329.656858:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177329.656862:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa7100.
00000020:00000040:1.0:1786177329.656863:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786177329.656865:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177329.656865:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177329.656866:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177329.656867:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:2.0:1786177329.656868:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab2400.
02000000:00000001:1.0:1786177329.656869:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177329.656871:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177329.656876:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b03400.
00000400:00000200:2.0:1786177329.656877:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177329.656883:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1786177329.656883:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177329.656889:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177329.656897:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856365312, portal 25
00000100:00100000:1.0:1786177329.656901:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177329.656902:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177329.656905:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751743360 : -114547957808256 : ffff97d1b9aa4380)
00000100:00000001:1.0:1786177329.656915:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1786177329.656916:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177329.656919:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177329.656928:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fb3d20af] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.656930:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177329.656936:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177329.656942:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000001:1.0:1786177329.656946:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1786177329.656948:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856365312, offset 0
00000400:00000200:2.0:1786177329.656955:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00080000:1.0:1786177329.656959:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177329.656965:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:2.0:1786177329.656965:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177329.656969:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177329.656971:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1786177329.656974:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000001:1.0:1786177329.656975:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177329.656982:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7552
00000400:00000200:2.0:1786177329.656982:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177329.656989:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177329.656991:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177329.656994:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177329.656997:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177329.657000:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1786177329.657002:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255025:255025:256:4294967295] 192.168.202.110@tcp LPNI seq info [255014:255014:8:4294967295]
00000100:00080000:1.0:1786177329.657005:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1786177329.657008:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1786177329.657013:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.657016:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177329.657019:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.657024:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:2.0:1786177329.657027:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000001:1.0:1786177329.657028:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177329.657031:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:1.0:1786177329.657035:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177329.657039:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000800:00000200:2.0:1786177329.657039:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000001:1.0:1786177329.657044:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:1.0:1786177329.657047:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000800:00000010:2.0:1786177329.657048:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000100:00000040:1.0:1786177329.657054:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.657059:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1786177329.657059:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000040:1.0:1786177329.657064:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000200:2.0:1786177329.657066:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:1.0:1786177329.657068:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177329.657073:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000800:00000200:2.1:1786177329.657077:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:1.0:1786177329.657080:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:2.1:1786177329.657082:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:1.0:1786177329.657085:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177329.657089:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000100:2.0:1786177329.657092:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
02000000:00000001:1.0:1786177329.657094:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000010:2.0:1786177329.657096:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
02000000:00000001:1.0:1786177329.657098:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177329.657102:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177329.657104:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1786177329.657105:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00000001:1.0:1786177329.657110:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177329.657114:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1786177329.657115:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177329.657120:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000100:00000010:1.0:1786177329.657121:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa5500.
00000400:00000010:2.0:1786177329.657124:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000020:00000040:1.0:1786177329.657126:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000001:2.0:1786177329.657130:0:2391:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:1.0:1786177329.657131:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177329.657133:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177329.657136:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177329.657137:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:2.0:1786177329.657137:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786177329.657140:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177329.657145:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b00800.
00000100:00000001:1.0:1786177329.657150:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.657151:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00100000:1.0:1786177329.657156:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1786177329.657159:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657183:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177329.657184:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177329.657188:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.657188:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:1.0:1786177329.657195:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bf052360] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657197:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177329.657203:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177329.657205:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1786177329.657207:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786177329.657209:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302229197
00000100:00080000:1.0:1786177329.657210:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1786177329.657212:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177329.657213:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000010:2.0:1786177329.657215:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25cc0.
00000100:00000001:1.0:1786177329.657216:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177329.657219:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7552
00000100:00000001:2.0:1786177329.657221:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000400:00000200:1.0:1786177329.657223:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177329.657227:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302229197
00000400:00000200:1.0:1786177329.657228:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177329.657232:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000010:2.0:1786177329.657232:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f257e0.
00000100:00000040:1.0:1786177329.657237:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177329.657238:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1786177329.657239:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177329.657243:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.657244:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:2.0:1786177329.657244:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000001:1.0:1786177329.657249:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177329.657250:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000100:00000001:2.0:1786177329.657250:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1786177329.657253:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177329.657256:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000100:00000001:2.0:1786177329.657256:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000020:00000001:1.0:1786177329.657257:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:1.0:1786177329.657260:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:1.0:1786177329.657263:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.657265:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:2.0:1786177329.657267:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177329/real 1786177329]  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786177329.657268:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.657272:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177329.657276:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:1.0:1786177329.657281:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177329.657284:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177329.657286:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177329.657289:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177329.657292:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177329.657295:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177329.657298:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1786177329.657302:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177329.657303:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1786177329.657304:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000010:1.0:1786177329.657305:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa5880.
00000400:00000001:2.0:1786177329.657307:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:1.0:1786177329.657308:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000400:00000010:2.0:1786177329.657311:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
02000000:00000001:1.0:1786177329.657312:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177329.657314:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177329.657316:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1786177329.657317:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
02000000:00000001:1.0:1786177329.657318:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177329.657321:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1786177329.657321:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177329.657326:0:2391:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000010:1.0:1786177329.657328:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b02c00.
00000100:00000001:1.0:1786177329.657331:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1786177329.657333:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177329.657336:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177329.657354:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177329.657357:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1786177329.657357:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786177329.657362:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcbacf3d] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657367:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177329.657369:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.657371:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177329.657375:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177329.657381:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177329.657382:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1786177329.657384:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177329.657389:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1786177329.657391:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177329.657393:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177329.657397:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786177329.657400:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:1.0:1786177329.657401:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7552
00000100:00000040:2.0:1786177329.657409:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1786177329.657410:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177329.657414:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177329.657417:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177329.657422:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.657426:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1786177329.657428:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:1.0:1786177329.657429:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.657435:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:2.0:1786177329.657435:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.657438:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000001:1.0:1786177329.657440:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:2.0:1786177329.657441:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:1.0:1786177329.657444:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000100:00000001:2.0:1786177329.657445:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000001:1.0:1786177329.657448:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:2.0:1786177329.657449:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:1.0:1786177329.657453:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000100:00080000:2.0:1786177329.657454:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000020:00000001:1.0:1786177329.657459:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00080000:2.0:1786177329.657461:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000020:00000001:1.0:1786177329.657464:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000001:2.0:1786177329.657466:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177329.657472:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:2.0:1786177329.657474:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177329.657475:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177329.657480:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177329.657494:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177329.657498:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:1.0:1786177329.657503:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177329.657506:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177329.657507:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856365312:192.168.202.110@tcp:250:kworker.0
02000000:00000001:1.0:1786177329.657543:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1786177329.657544:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.657548:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
02000000:00000001:1.0:1786177329.657549:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177329.657554:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177329.657554:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4380 x1872934856365312/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177329.657560:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177329.657563:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1786177329.657567:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177329.657570:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.657570:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
00000100:00000010:1.0:1786177329.657573:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa6300.
02000000:00000001:2.0:1786177329.657574:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000020:00000040:1.0:1786177329.657578:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000010:2.0:1786177329.657578:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab2400.
02000000:00000001:1.0:1786177329.657583:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177329.657583:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000001:1.0:1786177329.657585:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177329.657587:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000020:00000001:2.0:1786177329.657587:0:2391:0:(genops.c:1238:class_import_put()) Process entered
02000000:00000001:1.0:1786177329.657590:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000020:00000040:2.0:1786177329.657591:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1786177329.657592:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177329.657595:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177329.657597:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b00c00.
02000000:00000010:2.0:1786177329.657599:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00000.
00000100:00000001:1.0:1786177329.657601:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177329.657605:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00100000:1.0:1786177329.657607:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1786177329.657608:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000001:1.0:1786177329.657614:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177329.657616:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177329.657618:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a345583] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1786177329.657620:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4380.
00000100:00000001:1.0:1786177329.657626:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.657627:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177329.657630:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177329.657631:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7563)
00000100:00000001:2.0:1786177329.657634:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.657643:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177329.657645:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.657647:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177329.657649:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177329.657650:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177329.657656:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657663:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177329.657664:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177329.657666:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177329.657668:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177329.657671:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177329.657673:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856365440:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177329.657677:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177329.657679:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177329.657680:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177329.657682:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177329.657685:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177329.657687:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177329.657690:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177329.657693:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177329.657696:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177329.657699:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856365440, portal 10
00000100:00000001:2.0:1786177329.657701:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177329.657702:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751755008 : -114547957796608 : ffff97d1b9aa7100)
00000100:00000040:2.0:1786177329.657708:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657714:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177329.657716:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177329.657718:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177329.657720:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856365440, offset 0
00000400:00000200:2.0:1786177329.657724:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177329.657727:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177329.657731:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177329.657733:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177329.657735:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177329.657738:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177329.657740:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177329.657743:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255026:255026:256:4294967295] 192.168.202.110@tcp LPNI seq info [255015:255015:8:4294967295]
00000400:00000200:2.0:1786177329.657747:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177329.657753:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177329.657757:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177329.657760:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177329.657764:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177329.657767:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1786177329.657771:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177329.657773:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1786177329.657777:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177329.657779:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177329.657782:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177329.657785:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177329.657787:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177329.657789:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177329.657791:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177329.657794:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657802:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177329.657804:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657811:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.657812:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177329.657814:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177329.657818:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:2.0:1786177329.657820:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177329.657822:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302225418
00000100:00000001:2.0:1786177329.657825:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302229197
00000100:00000001:2.0:1786177329.657827:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302225418
00000100:00000001:2.0:1786177329.657829:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302229197
00000100:00000001:2.0:1786177329.657831:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177329.657832:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.657834:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177329.657837:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177329/real 1786177329]  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177329.657844:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177329.657845:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177329.657847:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177329.657849:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177329.657850:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177329.657852:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177329.657855:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177329.657862:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657867:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177329.657868:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.657869:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177329.657870:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177329.657872:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657875:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177329.657877:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657881:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177329.657883:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.657884:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177329.657885:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.657886:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177329.657887:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177329.657888:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177329.657890:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177329.657891:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177329.657893:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177329.657897:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856365440:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177329.657899:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177329.657901:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657905:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177329.657906:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177329.657907:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177329.657908:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177329.657909:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177329.657911:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856365568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177329.657912:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177329.657914:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177329.657915:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177329.657916:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177329.657918:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177329.657919:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177329.657921:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177329.657923:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177329.657924:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177329.657926:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856365568, portal 10
00000100:00000001:2.0:1786177329.657927:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177329.657928:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751747840 : -114547957803776 : ffff97d1b9aa5500)
00000100:00000040:2.0:1786177329.657932:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657936:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177329.657937:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177329.657938:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177329.657939:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856365568, offset 0
00000400:00000200:2.0:1786177329.657941:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177329.657943:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177329.657945:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177329.657947:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177329.657948:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177329.657950:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177329.657951:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177329.657953:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255027:255027:256:4294967295] 192.168.202.110@tcp LPNI seq info [255016:255016:8:4294967295]
00000400:00000200:2.0:1786177329.657955:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177329.657959:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177329.657961:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177329.657963:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177329.657965:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177329.657966:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1786177329.657969:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177329.657970:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1786177329.657973:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177329.657974:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177329.657977:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177329.657980:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177329.657982:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177329.657984:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177329.657986:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177329.657989:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.657996:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177329.657999:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658005:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.658007:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177329.658010:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177329.658013:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:2.0:1786177329.658016:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177329.658017:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302225543
00000100:00000001:2.0:1786177329.658020:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302229197
00000100:00000001:2.0:1786177329.658022:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302225543
00000100:00000001:2.0:1786177329.658024:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302229197
00000100:00000001:2.0:1786177329.658025:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177329.658026:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.658028:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177329.658030:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177329/real 1786177329]  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177329.658035:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177329.658036:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177329.658037:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177329.658038:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177329.658040:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177329.658041:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177329.658043:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177329.658048:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658052:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177329.658053:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658054:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177329.658056:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177329.658059:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658065:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177329.658069:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658075:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177329.658078:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658079:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177329.658081:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658083:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177329.658085:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177329.658087:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177329.658090:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177329.658092:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177329.658096:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177329.658103:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856365568:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177329.658107:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177329.658110:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658116:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177329.658118:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177329.658120:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177329.658121:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177329.658123:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177329.658126:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856365696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177329.658129:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177329.658130:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177329.658131:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177329.658132:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177329.658134:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177329.658136:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177329.658138:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177329.658140:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177329.658142:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177329.658144:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856365696, portal 4
00000100:00000001:2.0:1786177329.658145:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177329.658146:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751748736 : -114547957802880 : ffff97d1b9aa5880)
00000100:00000040:2.0:1786177329.658150:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658156:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177329.658158:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177329.658160:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177329.658162:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856365696, offset 0
00000400:00000200:2.0:1786177329.658166:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177329.658170:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177329.658173:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177329.658175:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177329.658177:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177329.658180:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177329.658182:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177329.658185:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255028:255028:256:4294967295] 192.168.202.110@tcp LPNI seq info [255017:255017:8:4294967295]
00000400:00000200:2.0:1786177329.658189:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177329.658194:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177329.658199:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177329.658201:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177329.658203:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177329.658205:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1786177329.658207:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177329.658209:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1786177329.658211:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177329.658212:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177329.658214:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177329.658216:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177329.658218:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177329.658219:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177329.658221:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177329.658223:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658228:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177329.658230:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658237:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.658238:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177329.658240:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177329.658242:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:2.0:1786177329.658244:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177329.658245:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302225418
00000100:00000001:2.0:1786177329.658247:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302229197
00000100:00000001:2.0:1786177329.658248:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302225418
00000100:00000001:2.0:1786177329.658249:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302229197
00000100:00000001:2.0:1786177329.658250:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177329.658252:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.658253:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177329.658255:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177329/real 1786177329]  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177329.658259:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177329.658259:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177329.658260:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177329.658262:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177329.658263:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177329.658264:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177329.658266:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177329.658270:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658273:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177329.658274:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658275:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177329.658276:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177329.658278:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658282:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177329.658284:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658288:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177329.658289:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658290:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177329.658291:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658291:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177329.658292:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177329.658294:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177329.658295:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177329.658296:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177329.658298:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177329.658302:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856365696:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177329.658304:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177329.658306:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658309:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177329.658310:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177329.658311:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177329.658312:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177329.658314:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177329.658315:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856365824:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177329.658317:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177329.658318:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177329.658319:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177329.658320:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177329.658322:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab0000.
02000000:00000001:2.0:1786177329.658323:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177329.658325:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177329.658326:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177329.658328:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177329.658329:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856365824, portal 4
00000100:00000001:2.0:1786177329.658330:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177329.658331:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751751424 : -114547957800192 : ffff97d1b9aa6300)
00000100:00000040:2.0:1786177329.658335:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658338:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177329.658340:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177329.658341:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000200:2.0:1786177329.658342:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856365824, offset 0
00000400:00000200:2.0:1786177329.658344:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177329.658346:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177329.658348:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177329.658349:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177329.658351:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177329.658352:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177329.658353:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177329.658355:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255029:255029:256:4294967295] 192.168.202.110@tcp LPNI seq info [255018:255018:8:4294967295]
00000400:00000200:2.0:1786177329.658357:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177329.658361:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177329.658364:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177329.658365:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
00000800:00000200:2.0:1786177329.658367:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177329.658369:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1786177329.658371:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177329.658373:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:2.0:1786177329.658375:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177329.658376:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:2.0:1786177329.658378:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177329.658380:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177329.658381:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:2.0:1786177329.658382:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:2.0:1786177329.658383:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177329.658385:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658389:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177329.658391:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658395:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.658396:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177329.658397:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177329.658399:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:2.0:1786177329.658400:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177329.658401:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302225426
00000100:00000001:2.0:1786177329.658403:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302229197
00000100:00000001:2.0:1786177329.658404:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302225426
00000100:00000001:2.0:1786177329.658405:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302229197
00000100:00000001:2.0:1786177329.658406:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177329.658407:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.658408:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177329.658410:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177329/real 1786177329]  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177329.658413:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177329.658414:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177329.658415:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177329.658416:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177329.658417:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177329.658418:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177329.658420:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177329.658425:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658428:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177329.658429:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658429:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177329.658430:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177329.658432:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658435:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177329.658437:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658441:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177329.658442:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658443:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177329.658443:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658444:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177329.658445:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177329.658447:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177329.658448:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177329.658450:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177329.658452:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177329.658456:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856365824:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177329.658458:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658459:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177329.658461:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa7100 x1872934856365440/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658465:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177329.658466:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177329.658467:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177329.658468:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177329.658469:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177329.658470:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177329.658471:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177329.658472:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03400.
02000000:00000001:2.0:1786177329.658474:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177329.658475:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177329.658476:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa7100.
00000100:00000001:2.0:1786177329.658477:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177329.658478:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658479:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177329.658481:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5500 x1872934856365568/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658498:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177329.658499:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177329.658500:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177329.658501:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177329.658502:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177329.658503:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177329.658504:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177329.658505:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00800.
02000000:00000001:2.0:1786177329.658506:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177329.658506:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177329.658507:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5500.
00000100:00000001:2.0:1786177329.658523:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177329.658525:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658526:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177329.658528:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5880 x1872934856365696/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658532:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177329.658533:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177329.658534:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177329.658535:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177329.658536:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177329.658537:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177329.658538:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177329.658538:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b02c00.
02000000:00000001:2.0:1786177329.658540:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177329.658540:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177329.658541:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5880.
00000100:00000001:2.0:1786177329.658542:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177329.658543:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658544:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177329.658545:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa6300 x1872934856365824/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177329.658549:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177329.658550:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177329.658550:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab0000.
02000000:00000001:2.0:1786177329.658552:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177329.658553:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177329.658553:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177329.658554:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177329.658555:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00c00.
02000000:00000001:2.0:1786177329.658556:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177329.658557:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177329.658557:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa6300.
00000100:00000001:2.0:1786177329.658558:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177329.658559:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658560:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177329.658562:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177329.658563:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177329.658564:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177329.658565:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177330.232611:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177330.232616:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177330.232620:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177330.232621:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177330.232623:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177330.232625:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177330.232670:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177330.232675:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177330.232678:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177330.680658:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177330.680666:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177330.680668:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177330.680673:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177330.680676:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177330.680679:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177330.680682:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177330.680685:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177330.680687:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177331.256618:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177331.256627:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177331.256633:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177331.256634:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177331.256637:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177331.256638:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177331.256686:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177331.256691:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177331.256695:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177331.704623:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177331.704631:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177331.704633:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177331.704639:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177331.704643:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177331.704646:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177331.704649:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177331.704652:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177331.704655:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177332.280630:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177332.280637:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177332.280639:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177332.280641:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177332.280645:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177332.280647:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177332.280649:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177332.280650:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177332.280653:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177332.280654:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177332.280655:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177332.280656:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177332.280657:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177332.280658:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177332.280659:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177332.280660:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177332.280662:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177332.280663:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177332.280723:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177332.280727:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177332.280730:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177332.728585:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177332.728596:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177332.728598:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177332.728602:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177332.728604:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177332.728607:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177332.728610:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177332.728611:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177332.728613:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177333.304602:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177333.304608:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177333.304613:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177333.304614:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177333.304617:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177333.304618:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177333.304672:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177333.304675:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177333.304677:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177333.496601:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302229197, at 4302229248
00000100:00000040:1.0:1786177333.496601:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302229197, at 4302229248
00000100:00000010:0.0:1786177333.496617:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25cc0.
00000100:00000010:1.0:1786177333.496622:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f257e0.
00000100:00000001:3.0:1786177333.752684:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177333.752694:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177333.752697:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177333.752705:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177333.752710:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177333.752715:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177333.752718:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177333.752722:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177333.752726:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177334.328694:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177334.328704:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177334.328711:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177334.328712:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177334.328715:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177334.328719:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177334.328722:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177334.328726:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177334.328730:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177334.328733:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177334.328735:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177334.328737:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177334.328803:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177334.328807:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177334.328810:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1786177334.776652:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177334.776652:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:1.0:1786177334.776663:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1786177334.776666:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177334.776666:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177334.776668:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177334.776670:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1786177334.776673:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:1.0:1786177334.776676:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177334.776676:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177334.776680:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:1.0:1786177334.776684:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7558
00000100:00000001:3.0:1786177334.776684:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177334.776687:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177334.776690:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177334.776693:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177334.776701:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177334.776705:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177334.776708:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177334.776711:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177334.776714:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177334.776716:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177334.776720:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177334.776723:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:1.0:1786177334.776726:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177334.776729:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:1.0:1786177334.776732:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:1.0:1786177334.776734:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:1.0:1786177334.776737:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177334.776744:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177334.776746:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177334.776748:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177334.776751:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:1.0:1786177334.776754:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177334.776757:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177334.776761:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177334.776764:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177334.776768:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.776770:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177334.776773:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177334.776785:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa6680.
00000020:00000040:1.0:1786177334.776788:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1786177334.776793:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177334.776795:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.776797:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177334.776801:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177334.776806:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177334.776815:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b01800.
00000100:00000001:1.0:1786177334.776823:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177334.776834:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177334.776845:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177334.776850:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177334.776855:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dced6303] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177334.776882:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177334.776886:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177334.776890:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1786177334.776891:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177334.776893:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177334.776896:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177334.776898:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7558
00000400:00000200:1.0:1786177334.776902:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177334.776905:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177334.776908:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177334.776911:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177334.776913:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177334.776916:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177334.776918:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177334.776921:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177334.776923:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:1.0:1786177334.776925:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177334.776927:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:2.0:1786177334.776928:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1786177334.776929:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:2.0:1786177334.776934:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1786177334.776935:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:1.0:1786177334.776939:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177334.776941:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177334.776943:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177334.776945:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:2.0:1786177334.776949:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1786177334.776950:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:1.0:1786177334.776955:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177334.776960:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177334.776962:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1786177334.776962:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786177334.776970:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177334.776976:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.776976:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177334.776980:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.776981:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177334.776988:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1786177334.776988:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177334.776993:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:1.0:1786177334.776994:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177334.776999:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.777000:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177334.777007:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa5c00.
00000100:00100000:2.0:1786177334.777007:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856365952:192.168.202.110@tcp:250:kworker.0
00000020:00000040:1.0:1786177334.777012:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:2.0:1786177334.777014:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177334.777016:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177334.777019:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.777021:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177334.777022:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177334.777023:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177334.777026:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177334.777027:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177334.777033:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177334.777035:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b03000.
00000100:00000001:1.0:1786177334.777040:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177334.777042:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177334.777046:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177334.777047:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1786177334.777056:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177334.777061:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1786177334.777063:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000100:00000001:1.0:1786177334.777067:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177334.777073:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d746f547] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1786177334.777074:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177334.777085:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856365952, portal 25
00000100:00000001:1.0:1786177334.777086:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177334.777091:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1786177334.777091:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177334.777095:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751752320 : -114547957799296 : ffff97d1b9aa6680)
00000100:00080000:1.0:1786177334.777097:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1786177334.777100:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177334.777103:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:2.0:1786177334.777106:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177334.777107:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177334.777113:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7558
00000400:00000200:1.0:1786177334.777122:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177334.777124:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:1.0:1786177334.777128:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:2.0:1786177334.777129:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:1.0:1786177334.777133:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:2.0:1786177334.777135:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2cc0.
00000100:00000040:1.0:1786177334.777141:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:2.0:1786177334.777141:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856365952, offset 0
00000100:00000001:1.0:1786177334.777146:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177334.777149:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177334.777152:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:2.0:1786177334.777155:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000001:1.0:1786177334.777156:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177334.777159:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:1.0:1786177334.777161:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177334.777164:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:1.0:1786177334.777166:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:2.0:1786177334.777170:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000001:1.0:1786177334.777171:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:1.0:1786177334.777177:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177334.777182:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1786177334.777182:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:1.0:1786177334.777187:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177334.777189:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:2.0:1786177334.777192:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:1.0:1786177334.777193:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:2.0:1786177334.777199:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:1.0:1786177334.777200:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177334.777204:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177334.777206:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:1.0:1786177334.777207:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1786177334.777211:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:1.0:1786177334.777213:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1786177334.777218:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255030:255030:256:4294967295] 192.168.202.110@tcp LPNI seq info [255019:255019:8:4294967295]
02000000:00000001:1.0:1786177334.777219:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.777224:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177334.777228:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1786177334.777228:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00000001:1.0:1786177334.777234:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.777236:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177334.777241:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa7800.
00000400:00000200:2.0:1786177334.777246:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177334.777254:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:1.0:1786177334.777255:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786177334.777260:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000010:2.0:1786177334.777261:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef100.
02000000:00000001:1.0:1786177334.777262:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.777266:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177334.777268:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177334.777270:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786177334.777272:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:1.0:1786177334.777277:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b02400.
00000800:00000200:2.1:1786177334.777279:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:1.0:1786177334.777283:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177334.777291:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177334.777298:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177334.777300:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.777303:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177334.777305:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e989e18] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1786177334.777313:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177334.777319:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302234317
00000100:00000001:1.0:1786177334.777321:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177334.777325:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:2.0:1786177334.777327:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f254e0.
00000100:00080000:1.0:1786177334.777329:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1786177334.777332:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177334.777334:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177334.777337:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1786177334.777337:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00080000:1.0:1786177334.777343:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7558
00000100:00000001:2.0:1786177334.777347:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302234317
00000400:00000200:1.0:1786177334.777348:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000010:2.0:1786177334.777354:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f251e0.
00000400:00000200:1.0:1786177334.777355:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177334.777360:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1786177334.777360:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:1.0:1786177334.777365:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177334.777366:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177334.777370:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1786177334.777372:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177334.777375:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177334.777377:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.777379:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:2.0:1786177334.777383:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1786177334.777384:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177334.777388:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:1.0:1786177334.777391:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177334.777393:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:1.0:1786177334.777395:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:1.0:1786177334.777397:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:1.0:1786177334.777400:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177334.777402:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177334.777403:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1786177334.777406:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000040:1.0:1786177334.777407:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177334.777410:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177334.777412:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:2.0:1786177334.777413:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177334.777415:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1786177334.777417:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00000040:2.0:1786177334.777423:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1786177334.777424:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177334.777430:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177334.777446:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177334.777450:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177334.777452:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1786177334.777454:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.777455:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.777458:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:2.0:1786177334.777461:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:1.0:1786177334.777463:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
02000000:00000001:2.0:1786177334.777465:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:1.0:1786177334.777468:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177334.777469:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.777472:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177334.777477:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366080:192.168.202.110@tcp:38:kworker.0
00000100:00000010:1.0:1786177334.777479:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa5f80.
00000020:00000040:1.0:1786177334.777485:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:2.0:1786177334.777485:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177334.777488:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177334.777491:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177334.777491:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177334.777494:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177334.777495:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1786177334.777499:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177334.777502:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:2.0:1786177334.777502:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1400.
02000000:00000001:1.0:1786177334.777507:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177334.777507:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177334.777554:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b03800.
00000400:00000200:2.0:1786177334.777557:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177334.777560:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1786177334.777563:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a73c8.
00000100:00100000:1.0:1786177334.777569:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1786177334.777569:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a26e8.
00000100:00000200:2.0:1786177334.777574:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856366080, portal 10
00000100:00000001:2.0:1786177334.777579:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177334.777581:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751749632 : -114547957801984 : ffff97d1b9aa5c00)
00000100:00000001:1.0:1786177334.777586:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177334.777590:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177334.777593:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1786177334.777598:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5f3cd3c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177334.777610:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1786177334.777617:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177334.777617:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177334.777622:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a22a8.
00000100:00080000:1.0:1786177334.777623:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:2.0:1786177334.777627:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856366080, offset 0
00000100:00080000:1.0:1786177334.777629:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1786177334.777632:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177334.777636:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1786177334.777636:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:1.0:1786177334.777641:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1786177334.777643:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:1.0:1786177334.777647:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7558
00000400:00000200:2.0:1786177334.777651:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177334.777656:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177334.777658:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177334.777663:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1786177334.777664:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:1.0:1786177334.777668:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1786177334.777670:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:1.0:1786177334.777675:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:2.0:1786177334.777678:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:1.0:1786177334.777679:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177334.777685:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000400:00000200:2.0:1786177334.777686:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255031:255031:255:4294967295] 192.168.202.110@tcp LPNI seq info [255020:255020:7:4294967295]
00000100:00000001:1.0:1786177334.777688:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177334.777694:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:2.0:1786177334.777697:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000020:00000040:1.0:1786177334.777698:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:1.0:1786177334.777700:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177334.777704:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:1.0:1786177334.777708:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:2.0:1786177334.777711:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000020:00000001:1.0:1786177334.777713:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:1.0:1786177334.777719:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:2.0:1786177334.777724:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:1.0:1786177334.777725:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177334.777729:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000010:2.0:1786177334.777731:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef200.
00000100:00000001:1.0:1786177334.777732:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177334.777736:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000800:00000200:2.0:1786177334.777739:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00080000:1.0:1786177334.777742:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177334.777747:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1786177334.777748:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:1.0:1786177334.777751:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1786177334.777754:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177334.777756:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1786177334.777760:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
02000000:00000001:1.0:1786177334.777761:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.777766:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:2.0:1786177334.777767:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000008:00100000:1.0:1786177334.777770:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:2.0:1786177334.777771:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302230538
00000008:00000001:1.0:1786177334.777776:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.777777:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302234317
00000100:00000001:1.0:1786177334.777780:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.777782:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302230538
00000100:00000010:1.0:1786177334.777787:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa4700.
00000100:00000001:2.0:1786177334.777787:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302234317
00000100:00000001:2.0:1786177334.777791:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1786177334.777793:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
00000100:00000001:2.0:1786177334.777795:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177334.777799:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1786177334.777800:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
02000000:00000001:1.0:1786177334.777802:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177334.777805:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:2.0:1786177334.777806:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1786177334.777808:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177334.777811:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177334.777817:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b03c00.
00000100:00000001:2.0:1786177334.777821:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177334.777823:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.777824:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177334.777829:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00100000:1.0:1786177334.777830:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1786177334.777832:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177334.777838:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177334.777846:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366208:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177334.777847:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177334.777851:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.777853:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177334.777856:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:1.0:1786177334.777858:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c2491d39] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177334.777859:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177334.777864:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177334.777869:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:1.0:1786177334.777871:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177334.777874:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177334.777878:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:1.0:1786177334.777879:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7568)
00000400:00000010:2.0:1786177334.777882:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7318.
00000400:00000010:2.0:1786177334.777887:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177334.777892:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856366208, portal 10
00000100:00000001:2.0:1786177334.777894:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177334.777895:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751756800 : -114547957794816 : ffff97d1b9aa7800)
00000100:00000040:2.0:1786177334.777903:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177334.777910:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177334.777913:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177334.777916:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2dd0.
00000100:00000200:2.0:1786177334.777918:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856366208, offset 0
00000400:00000200:2.0:1786177334.777922:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177334.777926:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177334.777929:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177334.777932:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177334.777934:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177334.777936:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177334.777938:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177334.777941:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255032:255032:254:4294967295] 192.168.202.110@tcp LPNI seq info [255021:255021:6:4294967295]
00000400:00000200:2.0:1786177334.777946:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177334.777954:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177334.777959:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177334.777963:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efe00.
00000800:00000200:2.0:1786177334.777967:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177334.777971:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177334.777975:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177334.777978:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:2.0:1786177334.777982:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177334.777983:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302230663
00000100:00000001:2.0:1786177334.777986:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302234317
00000100:00000001:2.0:1786177334.777988:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302230663
00000100:00000001:2.0:1786177334.777990:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302234317
00000100:00000001:2.0:1786177334.777992:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177334.777995:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.777997:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.778000:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177334.778003:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177334.778005:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1786177334.778007:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177334.778008:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177334.778009:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177334.778013:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177334.778020:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177334.778022:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177334.778024:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177334.778026:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177334.778028:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177334.778031:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366336:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177334.778049:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177334.778051:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177334.778053:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177334.778055:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177334.778058:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1c00.
02000000:00000001:2.0:1786177334.778060:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177334.778064:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177334.778067:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7dc0.
00000400:00000010:2.0:1786177334.778071:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2000.
00000100:00000200:2.0:1786177334.778073:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856366336, portal 4
00000100:00000001:2.0:1786177334.778075:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177334.778076:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751750528 : -114547957801088 : ffff97d1b9aa5f80)
00000100:00000040:2.0:1786177334.778082:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177334.778089:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177334.778091:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177334.778099:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2bb0.
00000100:00000200:2.0:1786177334.778102:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856366336, offset 0
00000400:00000200:2.0:1786177334.778108:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177334.778111:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177334.778114:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177334.778117:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177334.778120:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177334.778122:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177334.778125:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177334.778128:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255033:255033:253:4294967295] 192.168.202.110@tcp LPNI seq info [255022:255022:5:4294967295]
00000400:00000200:2.0:1786177334.778132:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177334.778138:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177334.778143:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177334.778147:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef000.
00000800:00000200:2.0:1786177334.778150:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177334.778154:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177334.778157:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177334.778161:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:2.0:1786177334.778164:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177334.778166:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302230538
00000100:00000001:2.0:1786177334.778168:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302234317
00000100:00000001:2.0:1786177334.778171:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302230538
00000100:00000001:2.0:1786177334.778174:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302234317
00000100:00000001:2.0:1786177334.778176:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177334.778178:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.778180:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177334.778183:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177334.778191:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177334.778192:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177334.778194:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177334.778196:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177334.778198:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177334.778201:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177334.778204:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177334.778205:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177334.778207:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177334.778209:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177334.778212:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1000.
02000000:00000001:2.0:1786177334.778214:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177334.778216:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177334.778218:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7ec8.
00000400:00000010:2.0:1786177334.778222:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a23b8.
00000100:00000200:2.0:1786177334.778225:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856366464, portal 4
00000100:00000001:2.0:1786177334.778227:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177334.778228:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751744256 : -114547957807360 : ffff97d1b9aa4700)
00000100:00000040:2.0:1786177334.778234:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177334.778245:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177334.778248:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177334.778250:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a27f8.
00000100:00000200:2.0:1786177334.778252:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856366464, offset 0
00000400:00000200:2.0:1786177334.778257:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177334.778261:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177334.778264:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177334.778266:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177334.778268:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177334.778270:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177334.778272:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177334.778275:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255034:255034:252:4294967295] 192.168.202.110@tcp LPNI seq info [255023:255023:4:4294967295]
00000400:00000200:2.0:1786177334.778280:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177334.778286:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177334.778290:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177334.778294:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833ef800.
00000800:00000200:2.0:1786177334.778298:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177334.778302:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:2.0:1786177334.778304:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177334.778307:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:2.0:1786177334.778311:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177334.778312:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302230546
00000100:00000001:2.0:1786177334.778315:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302234317
00000100:00000001:2.0:1786177334.778317:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302230546
00000100:00000001:2.0:1786177334.778319:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302234317
00000100:00000001:2.0:1786177334.778322:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177334.778325:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.778328:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.778331:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177334.778334:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177334.778335:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1786177334.778338:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177334.778339:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177334.778341:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177334.778343:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177335.352671:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177335.352681:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177335.352688:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177335.352689:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177335.352692:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177335.352693:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177335.352777:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177335.352783:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177335.352788:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177335.800773:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177335.800784:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177335.800786:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177335.800792:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177335.800795:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177335.800799:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177335.800801:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177335.800804:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177335.800808:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177336.376711:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177336.376721:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177336.376728:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177336.376730:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177336.376734:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177336.376736:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177336.376820:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177336.376825:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177336.376830:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.0:1786177336.696647:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:3.0:1786177336.696655:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:3.0:1786177336.696660:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:1.0:1786177336.696915:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:1.0:1786177336.696922:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1786177336.696949:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:1.0:1786177336.696954:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:1.0:1786177336.696958:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177336.824673:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177336.824680:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177336.824682:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177336.824687:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177336.824690:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177336.824694:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177336.824696:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177336.824699:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177336.824702:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177337.400650:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177337.400658:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177337.400665:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177337.400666:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177337.400670:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177337.400671:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177337.400859:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177337.400862:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177337.400865:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177337.849699:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177337.849707:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177337.849710:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177337.849715:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177337.849719:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177337.849723:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177337.849726:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177337.849729:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177337.849732:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:1.0:1786177337.851481:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.110:988
00000400:00000100:1.0:1786177337.851568:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.110@tcp at host 192.168.202.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1786177337.851593:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177337.851599:0:2381:0:(router.c:1791:lnet_notify()) 192.168.202.10@tcp notifying 192.168.202.110@tcp: down
00000400:00000200:1.0:1786177337.851604:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177337.851609:0:2381:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1786177337.851615:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177337.851619:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef100.
00000400:00000200:1.0:1786177337.851627:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1786177337.851631:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177337.851638:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2cc0
00000400:00000010:1.0:1786177337.851641:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2cc0.
00000100:00000001:1.0:1786177337.851647:0:2381:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177337.851657:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177337.851688:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177337.851693:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177337.851700:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177337.851702:0:2381:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1786177337.851713:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177337.851716:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef200.
00000400:00000200:1.0:1786177337.851720:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1786177337.851723:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177337.851726:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a22a8
00000400:00000010:1.0:1786177337.851728:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a22a8.
00000100:00000001:1.0:1786177337.851731:0:2381:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177337.851736:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177337.851744:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177337.851748:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177337.851754:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177337.851756:0:2381:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1786177337.851760:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177337.851763:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efe00.
00000400:00000200:1.0:1786177337.851766:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1786177337.851769:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177337.851771:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2dd0
00000400:00000010:1.0:1786177337.851773:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2dd0.
00000100:00000001:1.0:1786177337.851775:0:2381:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177337.851780:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177337.851788:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177337.851790:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177337.851796:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177337.851797:0:2381:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1786177337.851802:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177337.851804:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef000.
00000400:00000200:1.0:1786177337.851807:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1786177337.851810:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177337.851813:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2bb0
00000400:00000010:1.0:1786177337.851815:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2bb0.
00000100:00000001:1.0:1786177337.851817:0:2381:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177337.851821:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177337.851828:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177337.851831:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177337.851837:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177337.851839:0:2381:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1786177337.851843:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.110@tcp
00000800:00000010:1.0:1786177337.851846:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833ef800.
00000400:00000200:1.0:1786177337.851849:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1786177337.851851:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1786177337.851854:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a27f8
00000400:00000010:1.0:1786177337.851855:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a27f8.
00000100:00000001:1.0:1786177337.851858:0:2381:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177337.851862:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177337.851868:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177337.851871:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177337.851877:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177337.851878:0:2381:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177337.852009:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177337.852013:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177337.852015:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177337.852022:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177334/real 1786177337]  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177337.852033:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177337.852035:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177337.852038:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7dc0.
00000400:00000200:2.0:1786177337.852042:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2000
00000400:00000010:2.0:1786177337.852043:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2000.
00000100:00000001:2.0:1786177337.852046:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177337.852049:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177337.852058:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852066:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177337.852067:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852070:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177337.852072:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177337.852075:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852082:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177337.852087:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852094:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177337.852097:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852099:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177337.852100:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852102:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177337.852104:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177337.852108:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177337.852112:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177337.852115:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177337.852118:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177337.852132:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366336:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177337.852137:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177337.852140:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177334/real 1786177337]  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177337.852152:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177337.852153:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177337.852155:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7ec8.
00000400:00000200:2.0:1786177337.852157:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a23b8
00000400:00000010:2.0:1786177337.852159:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a23b8.
00000100:00000001:2.0:1786177337.852160:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177337.852163:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177337.852171:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852176:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177337.852177:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852179:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177337.852180:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177337.852183:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852189:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177337.852192:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852198:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177337.852200:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852202:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177337.852203:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852204:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177337.852206:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177337.852208:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177337.852211:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177337.852214:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177337.852218:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177337.852224:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366464:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177337.852227:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177337.852231:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177334/real 1786177337]  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177337.852237:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177337.852238:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177337.852240:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a73c8.
00000400:00000200:2.0:1786177337.852242:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a26e8
00000400:00000010:2.0:1786177337.852243:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a26e8.
00000100:00000001:2.0:1786177337.852245:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177337.852248:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177337.852256:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852261:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177337.852263:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852264:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177337.852266:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177337.852269:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852275:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177337.852278:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852284:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177337.852286:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852287:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177337.852288:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852290:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177337.852291:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177337.852293:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177337.852296:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177337.852298:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177337.852301:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177337.852307:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366080:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177337.852311:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177337.852314:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177334/real 1786177337]  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177337.852320:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177337.852321:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177337.852322:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7318.
00000400:00000200:2.0:1786177337.852325:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177337.852326:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177337.852328:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177337.852331:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177337.852339:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852344:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177337.852345:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852347:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177337.852348:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177337.852351:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852361:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177337.852364:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852369:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177337.852371:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852372:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177337.852373:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852375:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177337.852377:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177337.852379:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177337.852382:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177337.852384:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177337.852387:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177337.852393:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366208:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177337.852396:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177337.852398:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177334/real 1786177337]  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177337.852422:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177337.852423:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177337.852425:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177337.852427:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177337.852429:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177337.852431:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177337.852434:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177337.852445:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852450:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177337.852451:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852453:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177337.852454:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177337.852457:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852463:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177337.852466:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852471:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177337.852474:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852475:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177337.852476:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852477:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177337.852479:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177337.852480:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177337.852484:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177337.852486:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177337.852489:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177337.852496:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856365952:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177337.852499:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852502:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177337.852505:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5f80 x1872934856366336/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852559:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177337.852563:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177337.852566:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:2.0:1786177337.852570:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177337.852634:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177337.852637:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177337.852640:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177337.852643:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03800.
02000000:00000001:2.0:1786177337.852646:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177337.852649:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177337.852650:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5f80.
00000100:00000001:2.0:1786177337.852654:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177337.852655:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852657:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177337.852661:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4700 x1872934856366464/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852667:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177337.852668:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177337.852670:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1000.
02000000:00000001:2.0:1786177337.852672:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177337.852673:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177337.852674:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177337.852676:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177337.852677:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03c00.
02000000:00000001:2.0:1786177337.852681:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177337.852681:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177337.852683:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4700.
00000100:00000001:2.0:1786177337.852684:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177337.852686:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852687:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177337.852690:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa5c00 x1872934856366080/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852696:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177337.852697:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177337.852698:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1400.
02000000:00000001:2.0:1786177337.852700:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177337.852701:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177337.852703:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177337.852704:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177337.852705:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03000.
02000000:00000001:2.0:1786177337.852707:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177337.852708:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177337.852710:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa5c00.
00000100:00000001:2.0:1786177337.852712:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177337.852713:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852714:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177337.852717:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa7800 x1872934856366208/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852723:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177337.852723:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177337.852725:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177337.852727:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177337.852728:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177337.852729:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177337.852730:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177337.852732:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b02400.
02000000:00000001:2.0:1786177337.852734:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177337.852735:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177337.852736:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa7800.
00000100:00000001:2.0:1786177337.852739:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177337.852740:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852741:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177337.852744:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa6680 x1872934856365952/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177337.852749:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177337.852750:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177337.852752:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177337.852754:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177337.852755:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177337.852756:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1786177337.852758:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177337.852759:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b01800.
02000000:00000001:2.0:1786177337.852762:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177337.852763:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177337.852764:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa6680.
00000100:00000001:2.0:1786177337.852766:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177337.852767:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852770:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177337.852777:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177337.852779:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177337.852780:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177337.852782:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177338.424642:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177338.424648:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177338.424654:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177338.424655:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177338.424659:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177338.424660:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177338.424713:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177338.424715:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177338.424717:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177338.616648:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302234317, at 4302234368
00000100:00000010:1.0:1786177338.616664:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f251e0.
00000100:00000040:0.0:1786177338.616765:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302234317, at 4302234368
00000100:00000010:0.0:1786177338.616771:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f254e0.
00000100:00000001:3.0:1786177338.872694:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177338.872701:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177338.872703:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177338.872707:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177338.872710:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177338.872714:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177338.872717:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177338.872720:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177338.872722:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177339.448681:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177339.448688:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177339.448693:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177339.448694:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177339.448706:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177339.448708:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177339.448756:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177339.448759:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177339.448762:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786177339.896624:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177339.896632:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177339.896634:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177339.896637:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177339.896641:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177339.896647:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7563
00000400:00000200:2.0:1786177339.896655:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177339.896660:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177339.896663:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177339.896666:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.896668:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177339.896668:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000040:2.0:1786177339.896672:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177339.896674:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177339.896675:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000001:3.0:1786177339.896678:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177339.896679:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177339.896681:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000100:00100000:3.0:1786177339.896682:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000020:00000001:2.0:1786177339.896684:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177339.896686:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1786177339.896687:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000100:00000001:3.0:1786177339.896689:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1786177339.896690:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177339.896694:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000001:3.0:1786177339.896695:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1786177339.896699:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177339.896699:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177339.896703:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177339.896704:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177339.896707:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.896709:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177339.896711:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177339.896713:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177339.896715:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177339.896718:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177339.896721:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177339.896724:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.896726:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177339.896728:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177339.896733:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786177339.896735:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177339.896738:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177339.896739:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.896741:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177339.896743:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177339.896746:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177339.896752:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786177339.896757:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177339.896767:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177339.896778:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177339.896782:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177339.896786:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177339.896815:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177339.896818:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177339.896822:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177339.896823:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177339.896825:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177339.896827:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177339.896831:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7563
00000400:00000200:2.0:1786177339.896835:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177339.896838:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177339.896840:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177339.896843:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.896845:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177339.896847:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.896849:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177339.896851:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177339.896852:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177339.896854:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177339.896856:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177339.896857:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177339.896859:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177339.896862:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.896863:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177339.896865:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.896867:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177339.896869:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177339.896872:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177339.896874:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177339.896876:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177339.896881:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177339.896883:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.896885:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177339.896889:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177339.896892:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.896893:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177339.896897:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786177339.896899:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177339.896912:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177339.896913:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.896915:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177339.896917:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177339.896918:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177339.896922:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2400.
00000100:00000001:2.0:1786177339.896926:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177339.896931:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177339.896936:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177339.896938:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177339.896941:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177339.896949:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177339.896951:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177339.896954:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177339.896956:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177339.896958:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177339.896960:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177339.896962:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7563
00000400:00000200:2.0:1786177339.896966:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177339.896969:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177339.896977:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177339.896980:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.896982:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177339.896984:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.896986:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177339.896988:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177339.896989:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177339.896990:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177339.896992:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177339.896995:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786177339.896997:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786177339.897000:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897001:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177339.897003:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897005:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177339.897008:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786177339.897010:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897013:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177339.897015:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177339.897018:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177339.897020:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.897022:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177339.897024:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177339.897026:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.897028:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177339.897032:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786177339.897034:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177339.897037:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177339.897038:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.897040:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177339.897041:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177339.897043:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177339.897046:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1400.
00000100:00000001:2.0:1786177339.897050:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177339.897055:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177339.897062:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177339.897066:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177339.897071:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177339.897077:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177339.897080:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177339.897083:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177339.897084:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177339.897085:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177339.897089:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177339.897091:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7563
00000400:00000200:2.0:1786177339.897094:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177339.897096:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177339.897099:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177339.897101:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897102:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177339.897104:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897106:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177339.897108:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177339.897109:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177339.897110:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177339.897112:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177339.897114:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177339.897116:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786177339.897118:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897120:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177339.897122:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897123:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177339.897125:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177339.897127:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897129:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177339.897131:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177339.897134:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177339.897136:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.897138:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177339.897142:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177339.897143:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.897145:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177339.897148:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786177339.897150:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177339.897152:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177339.897153:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.897154:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177339.897155:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177339.897157:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177339.897160:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00000001:2.0:1786177339.897163:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177339.897167:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177339.897173:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177339.897174:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177339.897178:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177339.897184:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177339.897187:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177339.897190:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177339.897192:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177339.897193:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177339.897195:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177339.897198:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7563
00000400:00000200:2.0:1786177339.897200:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177339.897203:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177339.897206:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177339.897208:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897210:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177339.897212:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897214:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177339.897216:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177339.897217:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177339.897218:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177339.897220:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786177339.897222:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786177339.897223:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177339.897226:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897227:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177339.897228:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897230:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177339.897232:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786177339.897235:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177339.897236:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177339.897238:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177339.897241:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177339.897242:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.897244:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177339.897246:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177339.897248:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.897249:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177339.897252:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000020:00000040:2.0:1786177339.897254:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177339.897257:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177339.897258:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177339.897259:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177339.897260:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177339.897262:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177339.897265:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:2.0:1786177339.897268:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177339.897271:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177339.897283:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177339.897284:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177339.897288:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177339.897295:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177339.897300:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7573)
00000100:00000001:3.0:1786177339.897447:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177339.897451:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177339.897453:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177339.897459:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.897466:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177339.897467:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177339.897470:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177339.897473:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177339.897475:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177339.897478:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856366592:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177339.897481:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177339.897485:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177339.897487:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177339.897490:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177339.897494:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177339.897496:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177339.897501:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177339.897505:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177339.897535:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177339.897543:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856366592, portal 25
00000100:00000001:3.0:1786177339.897545:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177339.897547:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:3.0:1786177339.897552:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.897560:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177339.897562:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177339.897565:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177339.897569:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856366592, offset 0
00000400:00000200:3.0:1786177339.897576:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177339.897584:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177339.897590:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177339.897596:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177339.897599:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177339.897603:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177339.897606:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177339.897610:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255035:255035:256:4294967295] 192.168.202.110@tcp LPNI seq info [255024:255024:8:4294967295]
00000400:00000200:3.0:1786177339.897614:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177339.897624:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177339.897630:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177339.897635:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177339.897642:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177339.897645:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177339.897651:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177339.897654:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177339.897658:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177339.897660:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177339.897665:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177339.897669:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177339.897674:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177339.897678:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177339.897682:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177339.897686:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.897695:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177339.897698:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.897704:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177339.897706:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177339.897712:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177339.897715:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177339.897718:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302239438
00000100:00000010:3.0:1786177339.897723:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37c60.
00000100:00000001:3.0:1786177339.897731:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177339.897736:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302239438
00000100:00000010:3.0:1786177339.897739:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37f60.
00000100:00000001:3.0:1786177339.897741:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177339.897744:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177339.897746:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177339.897748:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177339.897752:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177339/real 1786177339]  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177339.897761:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177339.897763:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177339.897764:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177339.897767:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177339.897768:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177339.897770:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177339.897773:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177339.897780:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.897786:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177339.897788:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.897790:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177339.897792:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177339.897795:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.897801:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177339.897805:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.897811:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177339.897813:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.897815:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177339.897816:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.897819:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177339.897820:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177339.897824:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177339.897827:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177339.897830:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177339.897833:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177339.897841:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856366592:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177339.897845:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177339.897849:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.897855:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177339.897856:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177339.897858:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177339.897860:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177339.897862:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177339.897864:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177339.897867:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177339.897869:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177339.897870:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177339.897872:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177339.897878:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177339.897880:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177339.897884:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177339.897887:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177339.897890:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177339.897893:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856366720, portal 10
00000100:00000001:3.0:1786177339.897895:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177339.897897:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:3.0:1786177339.897904:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.897910:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177339.897912:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177339.897914:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177339.897917:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856366720, offset 0
00000400:00000200:3.0:1786177339.897921:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177339.897925:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177339.897929:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177339.897931:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177339.897934:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177339.897936:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177339.897938:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177339.897941:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255036:255036:256:4294967295] 192.168.202.110@tcp LPNI seq info [255025:255025:8:4294967295]
00000400:00000200:3.0:1786177339.897945:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177339.897951:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177339.897955:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177339.897960:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177339.897965:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177339.897967:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177339.897971:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177339.897973:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177339.897976:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177339.897978:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177339.897981:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177339.897984:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177339.897987:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177339.897988:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177339.897991:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177339.897994:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898006:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177339.898009:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898015:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177339.898016:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177339.898019:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177339.898024:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177339.898027:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177339.898029:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302235658
00000100:00000001:3.0:1786177339.898031:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302239438
00000100:00000001:3.0:1786177339.898033:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302235658
00000100:00000001:3.0:1786177339.898035:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302239438
00000100:00000001:3.0:1786177339.898037:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177339.898039:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177339.898040:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177339.898044:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177339/real 1786177339]  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177339.898050:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177339.898052:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177339.898053:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177339.898055:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177339.898056:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177339.898058:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177339.898061:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177339.898067:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898073:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177339.898075:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898076:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177339.898078:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177339.898081:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898087:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177339.898090:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898096:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177339.898098:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898100:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177339.898101:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898102:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177339.898104:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177339.898106:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177339.898109:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177339.898111:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177339.898114:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177339.898120:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366720:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177339.898124:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177339.898127:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898132:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177339.898134:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177339.898136:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177339.898137:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177339.898139:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177339.898141:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177339.898144:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177339.898146:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177339.898148:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177339.898149:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177339.898152:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177339.898154:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177339.898157:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177339.898159:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177339.898161:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177339.898164:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856366848, portal 10
00000100:00000001:3.0:1786177339.898166:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177339.898167:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:3.0:1786177339.898172:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898176:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177339.898179:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177339.898180:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177339.898183:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856366848, offset 0
00000400:00000200:3.0:1786177339.898186:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177339.898189:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177339.898193:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177339.898195:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177339.898197:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177339.898199:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177339.898201:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177339.898204:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255037:255037:256:4294967295] 192.168.202.110@tcp LPNI seq info [255026:255026:8:4294967295]
00000400:00000200:3.0:1786177339.898208:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177339.898213:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177339.898216:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177339.898219:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177339.898225:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177339.898227:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177339.898230:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177339.898232:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177339.898235:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177339.898236:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177339.898239:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177339.898242:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177339.898245:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177339.898246:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177339.898248:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177339.898251:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898257:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177339.898260:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898265:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177339.898266:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177339.898269:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177339.898271:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177339.898274:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177339.898275:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302235783
00000100:00000001:3.0:1786177339.898278:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302239438
00000100:00000001:3.0:1786177339.898280:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302235783
00000100:00000001:3.0:1786177339.898281:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302239438
00000100:00000001:3.0:1786177339.898283:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177339.898285:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177339.898286:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177339.898289:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177339/real 1786177339]  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177339.898296:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177339.898297:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177339.898299:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177339.898300:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177339.898302:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177339.898303:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177339.898306:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177339.898313:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898319:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177339.898321:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898323:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177339.898324:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177339.898326:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898332:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177339.898336:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898342:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177339.898344:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898345:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177339.898347:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898348:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177339.898350:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177339.898352:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177339.898354:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177339.898356:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177339.898360:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177339.898372:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366848:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177339.898376:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177339.898379:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898399:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177339.898400:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177339.898402:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177339.898403:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177339.898405:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177339.898407:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177339.898410:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177339.898412:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177339.898413:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177339.898414:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177339.898417:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177339.898419:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177339.898422:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177339.898424:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177339.898428:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177339.898431:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856366976, portal 4
00000100:00000001:3.0:1786177339.898433:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177339.898434:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000040:3.0:1786177339.898438:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898444:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177339.898445:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177339.898447:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177339.898449:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856366976, offset 0
00000400:00000200:3.0:1786177339.898452:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177339.898456:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177339.898459:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177339.898461:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177339.898464:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177339.898466:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177339.898468:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177339.898470:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255038:255038:256:4294967295] 192.168.202.110@tcp LPNI seq info [255027:255027:8:4294967295]
00000400:00000200:3.0:1786177339.898473:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177339.898478:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177339.898482:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177339.898484:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177339.898487:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177339.898489:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177339.898493:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177339.898495:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177339.898498:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177339.898499:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177339.898502:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177339.898504:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177339.898507:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177339.898541:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177339.898544:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177339.898547:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898553:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177339.898556:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898561:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177339.898562:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177339.898565:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177339.898568:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177339.898571:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177339.898573:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302235659
00000100:00000001:3.0:1786177339.898575:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302239438
00000100:00000001:3.0:1786177339.898577:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302235659
00000100:00000001:3.0:1786177339.898579:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302239438
00000100:00000001:3.0:1786177339.898580:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177339.898582:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177339.898584:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177339.898586:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177339/real 1786177339]  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177339.898593:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177339.898594:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177339.898595:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177339.898597:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177339.898599:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177339.898601:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177339.898604:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177339.898610:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898616:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177339.898617:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898618:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177339.898619:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177339.898621:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898627:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177339.898629:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898635:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177339.898637:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898639:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177339.898640:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898641:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177339.898643:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177339.898645:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177339.898648:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177339.898650:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177339.898653:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177339.898659:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856366976:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177339.898663:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177339.898666:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898673:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177339.898674:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177339.898676:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177339.898678:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177339.898680:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177339.898683:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856367104:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177339.898686:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177339.898688:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177339.898689:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177339.898691:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177339.898695:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177339.898697:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177339.898699:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177339.898702:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177339.898705:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177339.898707:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856367104, portal 4
00000100:00000001:3.0:1786177339.898711:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177339.898712:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:3.0:1786177339.898719:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898725:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177339.898727:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177339.898729:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177339.898732:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856367104, offset 0
00000400:00000200:3.0:1786177339.898735:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177339.898739:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177339.898742:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177339.898745:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177339.898747:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177339.898749:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177339.898751:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177339.898754:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255039:255039:256:4294967295] 192.168.202.110@tcp LPNI seq info [255028:255028:8:4294967295]
00000400:00000200:3.0:1786177339.898757:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177339.898762:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177339.898766:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177339.898769:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177339.898773:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177339.898775:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1786177339.898779:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177339.898782:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1786177339.898785:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177339.898787:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177339.898790:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177339.898793:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177339.898795:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177339.898797:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177339.898799:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177339.898802:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898814:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177339.898817:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898822:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177339.898823:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177339.898825:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177339.898828:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177339.898832:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177339.898833:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302235667
00000100:00000001:3.0:1786177339.898836:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302239438
00000100:00000001:3.0:1786177339.898838:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302235667
00000100:00000001:3.0:1786177339.898840:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302239438
00000100:00000001:3.0:1786177339.898841:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177339.898844:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177339.898845:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177339.898849:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177339/real 1786177339]  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177339.898857:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177339.898858:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177339.898859:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177339.898862:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177339.898863:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177339.898865:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177339.898868:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177339.898876:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898882:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177339.898884:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898885:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177339.898886:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177339.898890:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898897:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177339.898900:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898907:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177339.898909:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898910:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177339.898911:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898913:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177339.898915:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177339.898917:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177339.898920:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177339.898923:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177339.898927:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177339.898933:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856367104:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177339.898937:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898939:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177339.898942:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856366592/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898948:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177339.898951:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177339.898955:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:3.0:1786177339.898958:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177339.898966:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177339.898969:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177339.898972:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177339.898974:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:3.0:1786177339.898978:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177339.898980:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177339.898982:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:3.0:1786177339.898986:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177339.898988:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.898989:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177339.898993:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856366720/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.898997:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177339.898998:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177339.898999:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:3.0:1786177339.899001:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177339.899002:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177339.899002:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177339.899004:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177339.899004:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2400.
02000000:00000001:3.0:1786177339.899006:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177339.899007:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177339.899008:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:3.0:1786177339.899010:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177339.899011:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.899013:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177339.899016:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856366848/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.899022:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177339.899023:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177339.899024:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177339.899028:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177339.899029:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177339.899030:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177339.899032:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177339.899033:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1400.
02000000:00000001:3.0:1786177339.899035:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177339.899036:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177339.899037:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:3.0:1786177339.899039:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177339.899040:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.899042:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177339.899045:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856366976/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.899052:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177339.899053:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177339.899054:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177339.899056:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177339.899057:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177339.899059:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177339.899060:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177339.899062:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:3.0:1786177339.899063:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177339.899064:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177339.899065:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:3.0:1786177339.899066:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177339.899067:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.899068:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177339.899070:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856367104/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177339.899074:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177339.899075:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177339.899076:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177339.899077:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177339.899078:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177339.899079:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177339.899079:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177339.899080:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:3.0:1786177339.899081:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177339.899082:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177339.899082:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:3.0:1786177339.899083:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177339.899084:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.899086:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177339.899092:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177339.899093:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177339.899095:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177339.899096:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177340.472637:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177340.472643:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177340.472650:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177340.472651:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177340.472655:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177340.472656:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177340.473428:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177340.473433:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177340.473437:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177340.920732:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177340.920738:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177340.920741:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177340.920747:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177340.920750:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177340.920754:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177340.920757:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177340.920759:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177340.920762:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177341.496654:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177341.496660:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177341.496666:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177341.496667:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177341.496671:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177341.496672:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177341.496789:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177341.496792:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177341.496795:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177341.944594:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177341.944602:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177341.944605:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177341.944610:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177341.944613:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177341.944618:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177341.944620:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177341.944623:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177341.944626:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177342.520651:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177342.520670:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177342.520673:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177342.520675:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177342.520680:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177342.520682:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177342.520684:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177342.520686:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177342.520689:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177342.520691:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177342.520692:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177342.520693:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177342.520694:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177342.520695:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177342.520696:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177342.520697:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177342.520699:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177342.520700:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177342.520768:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177342.520772:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177342.520775:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177342.968671:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177342.968683:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177342.968685:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177342.968694:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177342.968698:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177342.968703:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177342.968706:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177342.968709:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177342.968712:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177343.545772:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177343.545779:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177343.545784:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177343.545785:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177343.545787:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177343.545788:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177343.545878:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177343.545883:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177343.545887:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177343.736621:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302239438, at 4302239488
00000100:00000010:1.0:1786177343.736637:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37f60.
00000100:00000040:0.0:1786177343.736706:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302239438, at 4302239488
00000100:00000010:0.0:1786177343.736711:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37c60.
00000100:00000001:3.0:1786177343.993470:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177343.993478:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177343.993480:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177343.993486:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177343.993490:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177343.993493:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177343.993496:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177343.993499:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177343.993502:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177344.568640:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177344.568647:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177344.568653:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177344.568654:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177344.568657:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177344.568660:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177344.568662:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177344.568665:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177344.568668:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177344.568671:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177344.568673:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177344.568675:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177344.568722:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177344.568725:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177344.568727:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177345.016628:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1786177345.016634:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177345.016634:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177345.016637:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177345.016645:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00080000:2.0:1786177345.016649:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177345.016651:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177345.016655:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1786177345.016657:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1786177345.016661:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177345.016663:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177345.016668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177345.016670:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177345.016678:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177345.016684:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7568
00000400:00000200:2.0:1786177345.016694:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177345.016701:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177345.016704:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177345.016707:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177345.016711:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177345.016713:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177345.016715:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177345.016718:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177345.016720:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177345.016722:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177345.016726:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177345.016728:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177345.016731:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177345.016735:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177345.016737:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177345.016738:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177345.016740:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177345.016744:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177345.016747:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177345.016750:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177345.016754:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177345.016758:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177345.016761:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177345.016764:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177345.016766:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177345.016778:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd500.
00000020:00000040:2.0:1786177345.016782:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177345.016787:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177345.016789:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177345.016793:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177345.016797:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177345.016801:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177345.016811:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0c00.
00000100:00000001:2.0:1786177345.016819:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177345.016830:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177345.016842:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177345.016847:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177345.016853:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085ac295a] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177345.016877:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177345.016880:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177345.016882:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177345.016884:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177345.016885:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177345.016887:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177345.016889:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7568
00000400:00000200:2.0:1786177345.016892:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177345.016894:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177345.016896:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177345.016899:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177345.016902:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177345.016904:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177345.016905:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177345.016908:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177345.016909:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177345.016911:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177345.016913:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177345.016915:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177345.016917:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177345.016920:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177345.016922:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177345.016924:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177345.016926:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177345.016928:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177345.016931:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177345.016933:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177345.016935:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177345.016941:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177345.016943:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177345.016945:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177345.016949:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177345.016959:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177345.016962:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177345.016965:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eded80.
00000020:00000040:2.0:1786177345.016968:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177345.016970:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177345.016971:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177345.016973:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177345.016974:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177345.016976:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177345.016980:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2000.
00000100:00000001:2.0:1786177345.016984:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177345.016989:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177345.016997:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177345.016999:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177345.017003:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e962af94] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.017008:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177345.017011:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.017014:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1786177345.017015:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177345.017019:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1786177345.017021:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177345.017024:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177345.017026:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:3.0:1786177345.017030:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177345.017031:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177345.017036:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7568
00000400:00000200:2.0:1786177345.017045:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177345.017049:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000400:00000200:2.0:1786177345.017051:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177345.017053:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177345.017056:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
02000000:00000001:3.0:1786177345.017059:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1786177345.017062:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177345.017065:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177345.017067:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1786177345.017071:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177345.017072:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177345.017077:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177345.017080:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00100000:3.0:1786177345.017080:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856367232:192.168.202.110@tcp:250:kworker.0
00000020:00000040:2.0:1786177345.017084:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177345.017088:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000100:00000001:3.0:1786177345.017090:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1786177345.017092:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177345.017096:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
02000000:00000001:3.0:1786177345.017096:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:2.0:1786177345.017101:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
02000000:00000001:3.0:1786177345.017101:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177345.017108:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:2.0:1786177345.017109:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177345.017114:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000010:3.0:1786177345.017115:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000040:2.0:1786177345.017116:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000001:3.0:1786177345.017121:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177345.017122:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177345.017126:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786177345.017129:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177345.017133:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177345.017133:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:2.0:1786177345.017138:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1786177345.017140:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
02000000:00000001:2.0:1786177345.017141:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177345.017146:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177345.017149:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177345.017151:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000400:00000010:3.0:1786177345.017151:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000008:00000001:2.0:1786177345.017157:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177345.017159:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177345.017162:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856367232, portal 25
00000100:00000010:2.0:1786177345.017164:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd180.
00000100:00000001:3.0:1786177345.017167:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000020:00000040:2.0:1786177345.017169:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000100:00000001:3.0:1786177345.017171:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454181632 : -114548255369984 : ffff97d1a7edd500)
02000000:00000001:2.0:1786177345.017174:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177345.017177:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177345.017180:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:3.0:1786177345.017182:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1786177345.017183:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177345.017186:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.017195:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000010:2.0:1786177345.017196:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab2800.
00000100:00000040:3.0:1786177345.017201:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177345.017203:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177345.017207:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00100000:2.0:1786177345.017211:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1786177345.017221:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856367232, offset 0
00000100:00000001:2.0:1786177345.017222:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177345.017227:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177345.017233:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000040:2.0:1786177345.017234:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f62e8a91] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1786177345.017245:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177345.017246:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177345.017251:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1786177345.017254:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00080000:2.0:1786177345.017256:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177345.017259:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177345.017261:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1786177345.017262:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1786177345.017268:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177345.017270:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00080000:2.0:1786177345.017274:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7568
00000400:00000200:3.0:1786177345.017278:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177345.017282:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177345.017285:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255040:255040:256:4294967295] 192.168.202.110@tcp LPNI seq info [255029:255029:8:4294967295]
00000400:00000200:3.0:1786177345.017290:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177345.017296:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177345.017304:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1786177345.017305:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1786177345.017309:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177345.017316:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000800:00000200:3.0:1786177345.017317:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000100:00000001:2.0:1786177345.017320:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177345.017323:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000800:00000010:3.0:1786177345.017325:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000100:00000001:2.0:1786177345.017326:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177345.017331:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177345.017332:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177345.017333:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177345.017348:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177345.017350:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177345.017352:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786177345.017356:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177345.017357:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177345.017363:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177345.017365:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000800:00000200:3.0:1786177345.017370:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000020:00000040:2.0:1786177345.017371:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177345.017378:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000800:00000200:3.0:1786177345.017379:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1786177345.017383:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177345.017388:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:3.1:1786177345.017389:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000001:2.0:1786177345.017393:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:3.1:1786177345.017395:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
02000000:00000001:2.0:1786177345.017397:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177345.017401:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000800:00000100:3.0:1786177345.017404:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000008:00100000:2.0:1786177345.017406:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177345.017410:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1786177345.017410:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000100:00000001:2.0:1786177345.017414:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177345.017419:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc700.
00000400:00000200:3.0:1786177345.017422:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:2.0:1786177345.017423:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177345.017428:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1786177345.017430:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:2.0:1786177345.017431:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177345.017436:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177345.017438:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1786177345.017440:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
02000000:00000001:2.0:1786177345.017441:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177345.017445:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
02000000:00000010:2.0:1786177345.017448:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1c00.
00000100:00000001:3.0:1786177345.017453:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1786177345.017454:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177345.017460:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177345.017464:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177345.017479:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1786177345.017481:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177345.017485:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177345.017486:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1786177345.017493:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f8ad1d2c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.017500:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.017504:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177345.017506:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.017537:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177345.017542:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1786177345.017544:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177345.017549:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302244558
00000100:00080000:2.0:1786177345.017550:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177345.017554:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177345.017557:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000010:3.0:1786177345.017558:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad373c0.
00000100:00000001:2.0:1786177345.017562:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177345.017567:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7568
00000100:00000001:3.0:1786177345.017568:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000400:00000200:2.0:1786177345.017574:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177345.017578:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302244558
00000400:00000200:2.0:1786177345.017579:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000010:3.0:1786177345.017584:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad370c0.
00000100:00080000:2.0:1786177345.017585:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1786177345.017591:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000040:2.0:1786177345.017592:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177345.017596:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177345.017598:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177345.017601:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177345.017602:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177345.017606:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1786177345.017607:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177345.017612:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00100000:3.0:1786177345.017613:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177345/real 1786177345]  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177345.017614:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177345.017618:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177345.017622:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786177345.017627:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000001:3.0:1786177345.017628:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:2.0:1786177345.017630:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000001:3.0:1786177345.017631:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177345.017635:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000100:00000040:2.0:1786177345.017637:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:3.0:1786177345.017639:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000100:00000001:2.0:1786177345.017643:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1786177345.017643:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177345.017648:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:2.0:1786177345.017649:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000200:3.0:1786177345.017653:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177345.017654:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177345.017660:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00000200:3.0:1786177345.017665:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1786177345.017667:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177345.017672:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177345.017677:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177345.017681:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177345.017683:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:2.0:1786177345.017684:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.017685:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177345.017688:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177345.017689:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000008:00100000:2.0:1786177345.017691:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1786177345.017692:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:2.0:1786177345.017697:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177345.017698:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177345.017700:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177345.017707:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7eddf80.
00000100:00000001:3.0:1786177345.017707:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177345.017711:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1786177345.017713:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177345.017719:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1786177345.017720:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:2.0:1786177345.017723:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.017724:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177345.017726:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1786177345.017726:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177345.017729:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1786177345.017730:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177345.017732:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000001:2.0:1786177345.017734:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.017734:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177345.017739:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000010:2.0:1786177345.017741:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab1000.
00000100:00080000:3.0:1786177345.017743:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177345.017747:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1786177345.017748:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177345.017750:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177345.017757:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1786177345.017760:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856367232:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177345.017767:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177345.017771:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177345.017776:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177345.017779:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177345.017781:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177345.017782:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177345.017783:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177345.017787:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177345.017789:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177345.017791:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000098a92f69] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177345.017793:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856367360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177345.017799:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177345.017801:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177345.017804:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177345.017806:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1786177345.017811:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177345.017811:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177345.017816:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177345.017818:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7578)
00000400:00000200:3.0:1786177345.017821:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177345.017824:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177345.017826:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177345.017829:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856367360, portal 10
00000100:00000001:3.0:1786177345.017830:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177345.017831:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187904 : -114548255363712 : ffff97d1a7eded80)
00000100:00000040:3.0:1786177345.017837:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.017842:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177345.017843:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177345.017845:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177345.017847:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856367360, offset 0
00000400:00000200:3.0:1786177345.017850:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177345.017853:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177345.017856:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177345.017859:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177345.017862:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177345.017864:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177345.017866:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177345.017869:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255041:255041:256:4294967295] 192.168.202.110@tcp LPNI seq info [255030:255030:8:4294967295]
00000400:00000200:3.0:1786177345.017873:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177345.017879:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177345.017884:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177345.017888:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177345.017892:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177345.017895:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786177345.017899:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177345.017902:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1786177345.017906:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177345.017908:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177345.017910:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177345.017914:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177345.017917:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177345.017919:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177345.017921:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177345.017925:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.017933:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177345.017935:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.017942:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.017943:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177345.017947:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177345.017950:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177345.017953:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177345.017955:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302240778
00000100:00000001:3.0:1786177345.017958:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302244558
00000100:00000001:3.0:1786177345.017960:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302240778
00000100:00000001:3.0:1786177345.017962:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302244558
00000100:00000001:3.0:1786177345.017965:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177345.017967:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.017968:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177345.017972:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177345/real 1786177345]  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177345.017986:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177345.017987:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177345.017990:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177345.017992:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177345.017994:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177345.017996:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177345.017999:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177345.018008:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018014:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177345.018016:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018018:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177345.018019:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177345.018023:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018029:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177345.018033:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018038:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177345.018040:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018041:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177345.018042:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018043:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177345.018044:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177345.018045:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177345.018047:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177345.018048:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177345.018051:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177345.018059:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856367360:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177345.018063:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018065:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177345.018068:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd500 x1872934856367232/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018075:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177345.018078:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177345.018081:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177345.018084:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177345.018086:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177345.018088:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177345.018090:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177345.018094:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0c00.
02000000:00000001:3.0:1786177345.018099:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177345.018100:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177345.018102:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd500.
00000100:00000001:3.0:1786177345.018104:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177345.018105:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018106:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177345.018108:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eded80 x1872934856367360/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018112:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177345.018113:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177345.018114:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177345.018116:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177345.018117:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177345.018118:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177345.018119:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177345.018120:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2000.
02000000:00000001:3.0:1786177345.018121:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177345.018122:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177345.018123:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eded80.
00000100:00000001:3.0:1786177345.018125:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177345.018126:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018128:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018135:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177345.018136:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.018137:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177345.018138:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177345.018139:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177345.018141:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018145:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177345.018146:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177345.018148:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177345.018149:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177345.018150:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177345.018152:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856367488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177345.018153:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177345.018155:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177345.018155:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177345.018157:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177345.018160:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177345.018162:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177345.018165:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177345.018167:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177345.018171:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177345.018174:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856367488, portal 10
00000100:00000001:3.0:1786177345.018175:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177345.018177:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454180736 : -114548255370880 : ffff97d1a7edd180)
00000100:00000040:3.0:1786177345.018182:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018189:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177345.018191:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177345.018193:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177345.018197:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856367488, offset 0
00000400:00000200:3.0:1786177345.018200:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177345.018204:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177345.018208:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177345.018210:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177345.018212:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177345.018214:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177345.018215:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177345.018217:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255042:255042:256:4294967295] 192.168.202.110@tcp LPNI seq info [255031:255031:8:4294967295]
00000400:00000200:3.0:1786177345.018219:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177345.018223:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177345.018226:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177345.018228:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177345.018230:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177345.018232:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786177345.018235:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177345.018237:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1786177345.018239:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177345.018240:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177345.018242:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177345.018244:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177345.018246:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177345.018247:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177345.018249:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177345.018251:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018257:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177345.018258:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018262:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.018263:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177345.018266:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177345.018268:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177345.018270:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177345.018271:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302240903
00000100:00000001:3.0:1786177345.018273:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302244558
00000100:00000001:3.0:1786177345.018274:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302240903
00000100:00000001:3.0:1786177345.018276:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302244558
00000100:00000001:3.0:1786177345.018277:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177345.018278:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.018279:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177345.018281:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177345/real 1786177345]  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177345.018286:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177345.018287:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177345.018288:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177345.018289:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177345.018290:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177345.018292:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177345.018294:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177345.018299:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018302:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177345.018303:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018303:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177345.018304:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177345.018306:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018309:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177345.018311:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018317:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177345.018318:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018319:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177345.018320:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018321:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177345.018322:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177345.018323:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177345.018325:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177345.018326:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177345.018328:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177345.018332:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856367488:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177345.018348:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177345.018351:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018355:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177345.018356:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177345.018358:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177345.018359:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177345.018360:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177345.018362:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856367616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177345.018364:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177345.018365:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177345.018366:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177345.018367:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177345.018369:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177345.018370:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177345.018372:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177345.018374:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177345.018376:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177345.018378:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856367616, portal 4
00000100:00000001:3.0:1786177345.018380:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177345.018381:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178048 : -114548255373568 : ffff97d1a7edc700)
00000100:00000040:3.0:1786177345.018385:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018389:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177345.018391:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177345.018393:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177345.018396:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856367616, offset 0
00000400:00000200:3.0:1786177345.018399:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177345.018403:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177345.018407:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177345.018409:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177345.018412:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177345.018414:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177345.018416:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177345.018419:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255043:255043:256:4294967295] 192.168.202.110@tcp LPNI seq info [255032:255032:8:4294967295]
00000400:00000200:3.0:1786177345.018423:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177345.018429:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177345.018433:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177345.018436:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177345.018439:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177345.018442:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786177345.018446:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177345.018448:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1786177345.018452:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177345.018454:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177345.018456:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177345.018459:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177345.018462:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177345.018464:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177345.018467:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177345.018470:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018477:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177345.018481:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018487:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.018488:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177345.018492:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177345.018495:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177345.018498:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177345.018500:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302240778
00000100:00000001:3.0:1786177345.018503:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302244558
00000100:00000001:3.0:1786177345.018505:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302240778
00000100:00000001:3.0:1786177345.018507:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302244558
00000100:00000001:3.0:1786177345.018529:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177345.018531:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.018533:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177345.018538:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177345/real 1786177345]  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177345.018545:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177345.018547:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177345.018548:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177345.018551:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177345.018552:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177345.018554:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177345.018557:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177345.018566:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018577:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177345.018579:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018580:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177345.018581:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177345.018585:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018591:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177345.018594:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018601:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177345.018602:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018603:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177345.018605:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018607:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177345.018608:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177345.018610:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177345.018613:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177345.018616:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177345.018620:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177345.018628:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856367616:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177345.018631:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177345.018634:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018640:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177345.018642:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177345.018644:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177345.018646:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177345.018648:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177345.018650:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856367744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177345.018653:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177345.018654:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177345.018656:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177345.018657:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177345.018660:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177345.018662:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177345.018665:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177345.018667:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177345.018670:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177345.018672:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856367744, portal 4
00000100:00000001:3.0:1786177345.018674:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177345.018675:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454184320 : -114548255367296 : ffff97d1a7eddf80)
00000100:00000040:3.0:1786177345.018680:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018687:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177345.018689:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177345.018691:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177345.018694:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856367744, offset 0
00000400:00000200:3.0:1786177345.018697:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177345.018701:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177345.018705:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177345.018707:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177345.018710:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177345.018712:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177345.018714:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177345.018717:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255044:255044:256:4294967295] 192.168.202.110@tcp LPNI seq info [255033:255033:8:4294967295]
00000400:00000200:3.0:1786177345.018721:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177345.018726:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177345.018731:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177345.018734:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177345.018738:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177345.018741:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1786177345.018745:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177345.018747:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1786177345.018751:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177345.018753:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177345.018755:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177345.018758:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177345.018761:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177345.018763:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177345.018765:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177345.018769:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018776:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177345.018779:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177370 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018784:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.018786:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177345.018789:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177345.018791:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177345.018794:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177345.018796:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302240787
00000100:00000001:3.0:1786177345.018798:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302244558
00000100:00000001:3.0:1786177345.018801:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302240787
00000100:00000001:3.0:1786177345.018803:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302244558
00000100:00000001:3.0:1786177345.018805:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177345.018806:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.018808:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177345.018812:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177345/real 1786177345]  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177345.018820:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177345.018821:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177345.018822:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177345.018825:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177345.018826:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177345.018828:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177345.018831:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177345.018840:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018846:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177345.018847:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018849:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177345.018850:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177345.018854:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018860:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177345.018864:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018870:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177345.018872:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018874:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177345.018875:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018877:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177345.018879:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177345.018881:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177345.018884:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177345.018887:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177345.018891:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177345.018899:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856367744:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177345.018902:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018904:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177345.018907:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd180 x1872934856367488/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018913:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177345.018915:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177345.018917:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177345.018920:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177345.018921:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177345.018922:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177345.018924:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177345.018926:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab2800.
02000000:00000001:3.0:1786177345.018928:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177345.018930:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177345.018931:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd180.
00000100:00000001:3.0:1786177345.018934:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177345.018935:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018936:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177345.018939:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc700 x1872934856367616/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018946:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177345.018947:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177345.018949:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177345.018951:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177345.018952:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177345.018953:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177345.018955:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177345.018956:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1c00.
02000000:00000001:3.0:1786177345.018958:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177345.018960:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177345.018961:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc700.
00000100:00000001:3.0:1786177345.018964:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177345.018965:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018967:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177345.018970:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7eddf80 x1872934856367744/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177370 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177345.018976:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177345.018977:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177345.018979:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177345.018981:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177345.018983:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177345.018984:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177345.018986:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177345.018987:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab1000.
02000000:00000001:3.0:1786177345.018989:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177345.018990:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177345.018992:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7eddf80.
00000100:00000001:3.0:1786177345.018994:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177345.018995:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.018997:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177345.019000:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177345.019002:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177345.019004:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177345.019005:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177345.592628:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177345.592635:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177345.592640:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177345.592642:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177345.592645:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177345.592646:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177345.592709:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177345.592714:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177345.592717:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177346.040624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177346.040633:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177346.040635:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177346.040642:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177346.040646:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177346.040650:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177346.040653:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177346.040655:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177346.040658:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177346.616592:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177346.616598:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177346.616603:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177346.616603:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177346.616605:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177346.616606:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177346.616693:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177346.616698:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177346.616702:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177347.064589:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177347.064597:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177347.064599:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177347.064605:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177347.064609:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177347.064612:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177347.064615:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177347.064617:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177347.064619:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177347.640648:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177347.640655:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177347.640659:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177347.640660:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177347.640663:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177347.640664:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177347.640722:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177347.640726:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177347.640728:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177348.088643:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177348.088651:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177348.088653:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177348.088658:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177348.088661:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177348.088664:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177348.088667:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177348.088670:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177348.088672:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177348.664604:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177348.664610:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177348.664615:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177348.664616:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177348.664619:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177348.664620:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177348.664695:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177348.664700:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177348.664704:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177348.856664:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302244558, at 4302244608
00000100:00000040:1.0:1786177348.856664:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302244558, at 4302244608
00000100:00000010:0.0:1786177348.856687:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad373c0.
00000100:00000010:1.0:1786177348.856689:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad370c0.
00000100:00000001:3.0:1786177349.112624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177349.112639:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177349.112642:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177349.112650:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177349.112654:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177349.112659:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177349.112662:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177349.112665:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177349.112669:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177349.688616:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177349.688638:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177349.688645:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177349.688647:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177349.688650:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177349.688652:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177349.688686:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177349.688690:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177349.688693:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177350.136723:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177350.136737:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177350.136740:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177350.136748:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177350.136753:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177350.136759:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.136762:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177350.136765:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.136768:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177350.137802:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177350.137815:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177350.137818:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177350.137824:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177350.137830:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177350.137842:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7573
00000400:00000200:0.0:1786177350.137855:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177350.137866:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177350.137871:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177350.137875:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.137879:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177350.137881:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.137885:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177350.137891:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177350.137895:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177350.137898:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177350.137903:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177350.137905:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177350.137908:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177350.137911:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.137913:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177350.137915:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.137917:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177350.137921:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177350.137925:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177350.137928:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177350.137932:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177350.137937:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177350.137942:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.137945:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177350.137948:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177350.137963:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418a80.
00000020:00000040:0.0:1786177350.137969:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177350.137974:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177350.137976:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.137981:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177350.137986:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177350.137995:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177350.138011:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4c00.
00000100:00000001:0.0:1786177350.138025:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177350.138044:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177350.138058:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177350.138065:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177350.138071:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000606c34cb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177350.138104:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177350.138108:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177350.138112:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177350.138113:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177350.138115:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177350.138118:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177350.138122:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7573
00000400:00000200:0.0:1786177350.138126:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177350.138129:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177350.138132:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177350.138135:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138137:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177350.138140:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138142:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177350.138144:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177350.138145:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177350.138148:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177350.138151:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177350.138153:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177350.138155:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177350.138159:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138161:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177350.138164:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138166:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177350.138168:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177350.138171:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138174:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177350.138176:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177350.138184:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177350.138186:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.138189:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177350.138196:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177350.138199:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.138202:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177350.138206:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418e00.
00000020:00000040:0.0:1786177350.138209:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177350.138211:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177350.138213:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.138214:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177350.138216:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177350.138218:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177350.138223:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5400.
00000100:00000001:0.0:1786177350.138227:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177350.138232:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177350.138239:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177350.138241:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177350.138244:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003428892b] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177350.138252:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177350.138254:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177350.138258:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177350.138259:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177350.138261:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177350.138264:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177350.138267:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7573
00000400:00000200:0.0:1786177350.138271:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177350.138274:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177350.138277:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177350.138280:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138282:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177350.138296:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138299:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177350.138301:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177350.138303:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177350.138304:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177350.138307:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177350.138309:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177350.138311:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177350.138314:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138316:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177350.138318:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138319:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177350.138322:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786177350.138324:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138327:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177350.138329:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177350.138331:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177350.138333:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.138335:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177350.138338:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177350.138341:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.138343:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177350.138349:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419180.
00000020:00000040:0.0:1786177350.138351:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177350.138354:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177350.138355:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.138357:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177350.138359:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177350.138360:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177350.138364:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6c00.
00000100:00000001:0.0:1786177350.138368:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177350.138372:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177350.138378:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177350.138380:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177350.138384:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005a89d8ab] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177350.138391:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177350.138393:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177350.138397:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177350.138398:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177350.138399:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177350.138402:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177350.138405:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7573
00000400:00000200:0.0:1786177350.138409:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177350.138412:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177350.138415:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177350.138418:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138420:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177350.138421:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138423:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177350.138425:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177350.138427:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177350.138429:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177350.138432:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177350.138434:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177350.138436:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177350.138440:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138442:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177350.138443:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138445:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177350.138448:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:0.0:1786177350.138450:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177350.138453:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177350.138454:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177350.138457:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177350.138459:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.138460:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177350.138463:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177350.138465:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.138467:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177350.138470:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b100.
00000020:00000040:0.0:1786177350.138472:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177350.138585:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177350.138590:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177350.138593:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177350.138600:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.138608:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177350.138611:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177350.138615:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177350.138618:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177350.138621:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177350.138624:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856367872:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177350.138629:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177350.138634:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177350.138638:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177350.138641:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177350.138646:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177350.138649:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177350.138655:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177350.138659:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177350.138675:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177350.138685:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856367872, portal 25
00000100:00000001:3.0:1786177350.138688:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177350.138690:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761657472 : -114547947894144 : ffff97d1ba418a80)
00000100:00000040:3.0:1786177350.138696:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.138704:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177350.138707:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177350.138711:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177350.138715:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856367872, offset 0
00000400:00000200:3.0:1786177350.138723:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177350.138734:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177350.138741:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177350.138747:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177350.138752:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177350.138757:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177350.138759:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177350.138764:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255045:255045:256:4294967295] 192.168.202.110@tcp LPNI seq info [255034:255034:8:4294967295]
00000400:00000200:3.0:1786177350.138769:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177350.138781:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177350.138788:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177350.138795:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177350.138803:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177350.138807:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177350.138813:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177350.138817:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177350.138823:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177350.138826:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177350.138835:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177350.138840:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177350.138847:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177350.138849:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177350.138855:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177350.138859:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.138869:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177350.138872:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.138879:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177350.138881:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177350.138889:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177350.138893:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177350.138896:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302249679
00000100:00000010:3.0:1786177350.138902:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad370c0.
00000100:00000001:3.0:1786177350.138911:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177350.138920:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302249679
00000100:00000010:3.0:1786177350.138924:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad373c0.
00000100:00000001:3.0:1786177350.138927:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177350.138931:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177350.138935:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177350.138937:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177350.138942:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177350/real 1786177350]  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177350.138953:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177350.138955:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177350.138957:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177350.138960:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177350.138962:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177350.138965:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177350.138969:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177350.138979:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.138987:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177350.138989:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.138992:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177350.138994:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177350.138998:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.139006:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177350.139010:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.139018:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177350.139021:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.139022:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177350.139023:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.139025:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177350.139027:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177350.139030:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177350.139033:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177350.139037:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177350.139041:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177350.139048:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856367872:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177350.139053:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177350.139056:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.139063:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177350.139064:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177350.139066:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177350.139068:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177350.139069:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177350.139072:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177350.139075:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177350.139077:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177350.139078:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177350.139080:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177350.139084:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177350.139085:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177350.139089:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177350.139091:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177350.139095:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177350.139098:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856368000, portal 10
00000100:00000001:3.0:1786177350.139099:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177350.139101:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761658368 : -114547947893248 : ffff97d1ba418e00)
00000100:00000040:3.0:1786177350.139107:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.139113:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177350.139115:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177350.139117:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177350.139120:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856368000, offset 0
00000400:00000200:3.0:1786177350.139123:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177350.139127:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177350.139131:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177350.139133:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177350.139136:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177350.139138:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177350.139140:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177350.139143:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255046:255046:256:4294967295] 192.168.202.110@tcp LPNI seq info [255035:255035:8:4294967295]
00000400:00000200:3.0:1786177350.139147:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177350.139153:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177350.139157:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177350.139161:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177350.139164:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177350.139166:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177350.139170:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177350.139173:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177350.139176:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177350.139178:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177350.139181:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177350.139184:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177350.139187:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177350.139189:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177350.139191:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177350.139194:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.139201:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177350.139204:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.139209:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177350.139211:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177350.139214:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177350.139219:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177350.139222:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177350.139224:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302245899
00000100:00000001:3.0:1786177350.139227:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302249679
00000100:00000001:3.0:1786177350.139230:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302245899
00000100:00000001:3.0:1786177350.139231:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302249679
00000100:00000001:3.0:1786177350.139233:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177350.139235:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177350.139237:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177350.139241:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177350/real 1786177350]  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177350.139248:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177350.139249:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177350.139251:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177350.139253:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177350.139255:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177350.139257:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177350.139260:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177350.139267:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.139274:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177350.139275:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.139277:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177350.139279:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177350.139282:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.139304:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177350.139308:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.139315:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177350.139318:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.139319:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177350.139321:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.139322:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177350.139324:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177350.139327:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177350.139330:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177350.139332:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177350.139336:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177350.139344:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368000:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177350.139348:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177350.139351:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.139357:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177350.139358:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177350.139360:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177350.139361:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177350.139363:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177350.139366:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177350.139369:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177350.139371:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177350.139373:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177350.139374:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177350.139378:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177350.139379:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177350.139383:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177350.139385:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177350.139388:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177350.139391:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856368128, portal 10
00000100:00000001:3.0:1786177350.139393:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177350.139394:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761659264 : -114547947892352 : ffff97d1ba419180)
00000100:00000040:3.0:1786177350.139400:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.139406:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177350.139408:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177350.139410:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177350.139412:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856368128, offset 0
00000400:00000200:3.0:1786177350.139416:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177350.139420:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177350.139423:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177350.139426:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177350.139429:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177350.139431:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177350.139434:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177350.139436:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255047:255047:256:4294967295] 192.168.202.110@tcp LPNI seq info [255036:255036:8:4294967295]
00000400:00000200:3.0:1786177350.139440:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177350.139446:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177350.139451:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177350.139454:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177350.139458:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177350.139461:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177350.139464:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177350.139467:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177350.139471:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177350.139472:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
02000000:00000001:0.0:1786177350.140631:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177350.140634:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.140638:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177350.140642:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177350.140647:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177350.140658:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5c00.
00000100:00000001:0.0:1786177350.140674:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177350.140685:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177350.140697:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177350.140702:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177350.140708:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000fa632fb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177350.140722:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177350.140726:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177350.140731:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177350.140734:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177350.140736:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177350.140740:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177350.140743:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7573
00000400:00000200:0.0:1786177350.140750:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177350.140755:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177350.140759:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177350.140763:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.140767:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177350.140769:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.140772:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177350.140776:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177350.140778:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177350.140781:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177350.140785:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177350.140788:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:0.0:1786177350.140790:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:0.0:1786177350.140793:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.140795:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177350.140797:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177350.140800:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177350.140803:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177350.140806:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177350.140808:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177350.140811:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177350.140819:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177350.140822:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.140825:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177350.140829:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177350.140832:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.140835:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177350.140840:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b480.
00000020:00000040:0.0:1786177350.140844:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177350.140846:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177350.140847:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177350.140849:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177350.140850:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177350.140852:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177350.140856:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7000.
00000100:00000001:0.0:1786177350.140859:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177350.140863:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177350.140871:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177350.140873:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177350.140877:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001215e9e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177350.140884:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177350.140891:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7583)
00000400:00000200:3.0:1786177350.141005:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177350.141012:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177350.141019:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177350.141023:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177350.141028:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177350.141033:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.141042:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177350.141046:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.141052:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177350.141054:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177350.141064:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177350.141071:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177350.141074:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177350.141077:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302246026
00000100:00000001:3.0:1786177350.141082:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302249679
00000100:00000001:3.0:1786177350.141084:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302246026
00000100:00000001:3.0:1786177350.141086:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302249679
00000100:00000001:3.0:1786177350.141089:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177350.141092:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177350.141096:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177350.141101:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177350/real 1786177350]  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177350.141111:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177350.141113:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177350.141116:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177350.141119:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177350.141121:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177350.141124:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177350.141127:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177350.141136:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.141144:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177350.141146:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.141149:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177350.141151:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177350.141155:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.141163:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177350.141168:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.141175:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177350.141179:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.141181:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177350.141182:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.141185:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177350.141188:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177350.141191:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177350.141196:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177350.141200:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177350.141204:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177350.141213:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368128:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177350.141219:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.141222:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177350.141226:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418a80 x1872934856367872/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.141234:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177350.141237:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177350.141241:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177350.141245:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177350.141247:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177350.141249:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177350.141252:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177350.141255:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4c00.
02000000:00000001:3.0:1786177350.141260:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177350.141263:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177350.141265:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418a80.
00000100:00000001:3.0:1786177350.141271:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177350.141272:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.141274:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177350.141278:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418e00 x1872934856368000/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.141299:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177350.141301:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177350.141302:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177350.141305:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177350.141306:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177350.141308:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177350.141309:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177350.141311:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5400.
02000000:00000001:3.0:1786177350.141314:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177350.141315:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177350.141316:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418e00.
00000100:00000001:3.0:1786177350.141319:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177350.141321:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.141322:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177350.141326:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419180 x1872934856368128/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.141332:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177350.141333:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177350.141334:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177350.141337:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177350.141338:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177350.141339:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177350.141342:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177350.141343:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6c00.
02000000:00000001:3.0:1786177350.141346:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177350.141347:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177350.141348:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419180.
00000100:00000001:3.0:1786177350.141350:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177350.141351:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.141355:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.141366:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177350.141369:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177350.141372:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177350.141375:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177350.141377:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177350.141382:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.141390:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177350.141393:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177350.141396:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177350.141399:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177350.141402:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177350.141406:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177350.141409:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177350.141414:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177350.141417:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177350.141420:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177350.141429:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177350.141431:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177350.141438:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177350.141441:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177350.141447:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177350.141454:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856368256, portal 4
00000100:00000001:3.0:1786177350.141457:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177350.141458:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761667328 : -114547947884288 : ffff97d1ba41b100)
00000100:00000040:3.0:1786177350.141466:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.144023:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177350.144038:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177350.144057:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177350.144084:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856368256, offset 0
00000400:00000200:3.0:1786177350.144098:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177350.144123:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177350.144139:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177350.144148:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177350.144155:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177350.144161:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177350.144164:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177350.144169:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255048:255048:256:4294967295] 192.168.202.110@tcp LPNI seq info [255037:255037:8:4294967295]
00000400:00000200:3.0:1786177350.144178:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177350.144193:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177350.144202:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177350.144212:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177350.144224:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177350.144231:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177350.144238:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177350.144243:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177350.144248:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177350.144252:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177350.144264:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177350.144270:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177350.144279:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177350.144283:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177350.144300:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177350.144310:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.144325:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177350.144329:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.144342:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177350.144344:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177350.144353:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177350.144359:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177350.144363:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177350.144367:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302245904
00000100:00000001:3.0:1786177350.144373:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302249679
00000100:00000001:3.0:1786177350.144375:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302245904
00000100:00000001:3.0:1786177350.144378:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302249679
00000100:00000001:3.0:1786177350.144381:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177350.144385:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177350.144389:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177350.144394:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177350/real 1786177350]  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177350.144405:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177350.144408:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177350.144411:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177350.144414:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177350.144416:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177350.144419:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177350.144423:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177350.144431:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.144438:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177350.144440:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.144443:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177350.144446:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177350.144450:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.144458:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177350.144465:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.145423:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177350.145439:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.145441:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177350.145444:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.145448:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177350.145452:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177350.145458:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177350.145470:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177350.145571:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177350.145582:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177350.145594:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368256:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177350.145601:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177350.145606:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.145613:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177350.145617:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177350.145621:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177350.145626:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177350.145629:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177350.145632:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177350.145637:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177350.145643:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177350.145648:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177350.145651:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177350.145673:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177350.145675:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177350.145683:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177350.145688:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177350.145695:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177350.145704:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856368384, portal 4
00000100:00000001:3.0:1786177350.145706:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177350.145708:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761668224 : -114547947883392 : ffff97d1ba41b480)
00000100:00000040:3.0:1786177350.145717:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.145724:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177350.145726:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177350.145729:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177350.145733:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856368384, offset 0
00000400:00000200:3.0:1786177350.145740:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177350.145750:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177350.145758:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177350.145763:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177350.145767:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177350.145771:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177350.145774:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177350.145778:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255049:255049:256:4294967295] 192.168.202.110@tcp LPNI seq info [255038:255038:8:4294967295]
00000400:00000200:3.0:1786177350.145783:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177350.145793:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177350.145799:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177350.145805:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177350.145812:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177350.145816:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1786177350.145822:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177350.145826:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1786177350.145830:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177350.145834:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177350.145843:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177350.145848:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177350.145855:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177350.145857:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177350.145863:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177350.145867:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.145877:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177350.145880:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177375 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.145887:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177350.145889:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177350.145896:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177350.145901:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177350.145904:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177350.145906:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302245914
00000100:00000001:3.0:1786177350.145910:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302249679
00000100:00000001:3.0:1786177350.145913:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302245914
00000100:00000001:3.0:1786177350.145915:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302249679
00000100:00000001:3.0:1786177350.145917:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177350.145920:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177350.145923:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177350.145926:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177350/real 1786177350]  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177350.145935:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177350.145937:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177350.145939:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177350.145942:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177350.145944:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177350.145947:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177350.145950:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177350.145957:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.145964:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177350.145965:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.145967:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177350.145968:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177350.145972:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.145978:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177350.145983:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.145989:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177350.145991:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.145993:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177350.145994:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.145995:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177350.145997:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177350.146000:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177350.146004:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177350.146006:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177350.146009:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177350.146016:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368384:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177350.146020:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.146023:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177350.146027:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b100 x1872934856368256/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.146033:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177350.146035:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177350.146038:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177350.146042:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177350.146044:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177350.146047:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177350.146050:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177350.146053:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5c00.
02000000:00000001:3.0:1786177350.146058:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177350.146059:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177350.146061:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b100.
00000100:00000001:3.0:1786177350.146065:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177350.146066:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.146068:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177350.146071:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b480 x1872934856368384/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177375 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177350.146077:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177350.146078:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177350.146079:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177350.146081:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177350.146082:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177350.146084:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177350.146085:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177350.146087:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7000.
02000000:00000001:3.0:1786177350.146089:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177350.146090:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177350.146091:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b480.
00000100:00000001:3.0:1786177350.146093:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177350.146095:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.146098:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177350.146107:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177350.146109:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177350.146112:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177350.146114:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177350.712733:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177350.712748:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177350.712755:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177350.712756:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177350.712758:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177350.712759:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177350.712865:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177350.712872:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177350.712877:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177351.160709:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177351.160721:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177351.160724:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177351.160732:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177351.160737:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177351.160741:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177351.160745:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177351.160749:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177351.160752:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177351.737433:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177351.737445:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177351.737448:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177351.737453:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177351.737459:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177351.737463:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177351.737465:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177351.737468:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177351.737473:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177351.737474:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177351.737475:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177351.737477:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177351.737479:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177351.737480:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177351.737482:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177351.737484:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177351.737487:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177351.737488:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177351.737620:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177351.737625:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177351.737629:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177352.184666:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177352.184677:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177352.184680:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177352.184687:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177352.184692:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177352.184697:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177352.184700:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177352.184704:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177352.184707:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177352.760669:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177352.760677:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177352.760682:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177352.760683:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177352.760686:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177352.760686:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177352.760755:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177352.760760:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177352.760770:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177353.208619:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177353.208627:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177353.208629:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177353.208635:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177353.208638:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177353.208641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177353.208643:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177353.208646:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177353.208648:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177353.784727:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177353.784737:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177353.784744:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177353.784746:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177353.784750:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177353.784751:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177353.784997:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177353.785001:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177353.785005:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177353.976587:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302249679, at 4302249728
00000100:00000010:0.0:1786177353.976605:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad370c0.
00000100:00000040:1.0:1786177353.976679:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302249679, at 4302249728
00000100:00000010:1.0:1786177353.976685:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad373c0.
00000100:00000001:3.0:1786177354.232671:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177354.232682:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177354.232684:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177354.232692:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177354.232697:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177354.232701:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177354.232714:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177354.232716:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177354.232719:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177354.808634:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177354.808643:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177354.808650:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177354.808651:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177354.808655:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177354.808658:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177354.808661:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177354.808664:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177354.808668:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177354.808672:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177354.808674:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177354.808676:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177354.809728:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177354.809734:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177354.809738:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786177355.256649:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177355.256653:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177355.256660:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1786177355.256661:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177355.256664:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177355.256665:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177355.256669:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1786177355.256671:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177355.256674:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177355.256676:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1786177355.256681:0:62275:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7578
00000100:00000001:3.0:1786177355.256682:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.256688:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000400:00000200:0.0:1786177355.256692:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177355.256692:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1786177355.256699:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177355.256704:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177355.256712:0:62275:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177355.256717:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.256721:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177355.256724:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.256727:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177355.256731:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177355.256734:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177355.256737:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177355.256741:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177355.256744:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177355.256747:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177355.256751:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.256753:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177355.256756:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.256758:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177355.256761:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177355.256765:0:62275:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177355.256767:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177355.256771:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177355.256774:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177355.256778:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177355.256780:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177355.256783:0:62275:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177355.256794:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41ad80.
00000020:00000040:0.0:1786177355.256799:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177355.256804:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177355.256805:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177355.256809:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177355.256812:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177355.256817:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177355.256827:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7c00.
00000100:00000001:0.0:1786177355.256835:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177355.256845:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177355.256852:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177355.256857:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177355.256860:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034e2c676] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177355.256878:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177355.256881:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177355.256883:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177355.256884:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177355.256886:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177355.256888:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177355.256891:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7578
00000400:00000200:0.0:1786177355.256895:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177355.256898:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177355.256901:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177355.256904:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.256906:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177355.256910:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.256912:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177355.256915:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177355.256916:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000100:00000001:3.0:1786177355.256918:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1786177355.256919:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177355.256924:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000100:00000001:3.0:1786177355.256924:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1786177355.256927:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000001:3.0:1786177355.256929:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1786177355.256933:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177355.256940:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000040:3.0:1786177355.256940:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177355.256944:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177355.256949:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.256953:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177355.256956:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177355.256958:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177355.256961:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177355.256965:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177355.256971:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1786177355.256973:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177355.256974:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177355.256979:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177355.256982:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1786177355.256983:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
02000000:00000001:3.0:1786177355.256988:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:0.0:1786177355.256989:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1786177355.256994:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1786177355.256995:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177355.256999:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177355.257002:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856368512:192.168.202.110@tcp:250:kworker.0
00000100:00000010:0.0:1786177355.257003:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41aa00.
00000020:00000040:0.0:1786177355.257008:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000100:00000001:3.0:1786177355.257011:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1786177355.257012:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177355.257014:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177355.257016:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177355.257018:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177355.257018:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1786177355.257020:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177355.257023:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177355.257025:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4800.
02000000:00000001:3.0:1786177355.257028:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1786177355.257030:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177355.257036:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
00000100:00100000:0.0:1786177355.257038:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1786177355.257040:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177355.257050:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1786177355.257051:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177355.257055:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1786177355.257058:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000100:00000040:0.0:1786177355.257061:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da0255a4] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1786177355.257068:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000001:0.0:1786177355.257080:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1786177355.257080:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856368512, portal 25
00000100:00080000:0.0:1786177355.257083:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177355.257085:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1786177355.257088:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177355.257089:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761666432 : -114547947885184 : ffff97d1ba41ad80)
00000100:00000001:0.0:1786177355.257091:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177355.257095:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177355.257100:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:3.0:1786177355.257101:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1786177355.257105:0:62275:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7578
00000400:00000200:0.0:1786177355.257114:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177355.257117:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:0.0:1786177355.257120:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1786177355.257125:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00080000:0.0:1786177355.257126:0:62275:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1786177355.257130:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000040:0.0:1786177355.257132:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:3.0:1786177355.257136:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856368512, offset 0
00000100:00000001:0.0:1786177355.257137:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177355.257142:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.257145:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177355.257148:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:3.0:1786177355.257150:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000020:00000040:0.0:1786177355.257151:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177355.257154:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177355.257157:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177355.257159:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177355.257161:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000400:00000200:3.0:1786177355.257164:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000040:0.0:1786177355.257168:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.257173:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1786177355.257176:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:0.0:1786177355.257177:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.257183:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177355.257186:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000400:00000200:3.0:1786177355.257187:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:0.0:1786177355.257191:0:62275:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:3.0:1786177355.257195:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000100:00000001:0.0:1786177355.257197:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177355.257202:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1786177355.257202:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1786177355.257206:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1786177355.257210:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
02000000:00000001:0.0:1786177355.257212:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177355.257215:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:3.0:1786177355.257217:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255050:255050:256:4294967295] 192.168.202.110@tcp LPNI seq info [255039:255039:8:4294967295]
00000008:00100000:0.0:1786177355.257219:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177355.257226:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177355.257230:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177355.257231:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177355.257262:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000100:00000010:0.0:1786177355.257264:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419500.
00000020:00000040:0.0:1786177355.257269:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000800:00000200:3.0:1786177355.257273:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
02000000:00000001:0.0:1786177355.257274:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177355.257277:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177355.257280:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177355.257282:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000010:3.0:1786177355.257282:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
02000000:00000001:0.0:1786177355.257285:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177355.257292:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4000.
00000800:00000200:3.0:1786177355.257293:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177355.257297:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1786177355.257299:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00100000:0.0:1786177355.257305:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1786177355.257313:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177355.257315:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177355.257318:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1786177355.257320:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:0.0:1786177355.257330:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b6718a9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:3.0:1786177355.257330:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177355.257336:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000100:00000001:0.0:1786177355.257343:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177355.257346:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1786177355.257346:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:0.0:1786177355.257353:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:0.0:1786177355.257356:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1786177355.257356:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:0.0:1786177355.257360:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177355.257365:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1786177355.257365:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000100:00080000:0.0:1786177355.257369:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7578
00000400:00000010:3.0:1786177355.257371:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000400:00000200:0.0:1786177355.257375:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177355.257378:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1786177355.257380:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:0.0:1786177355.257381:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177355.257385:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000200:3.0:1786177355.257389:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177355.257398:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177355.257402:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1786177355.257403:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.257408:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:3.0:1786177355.257411:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1786177355.257412:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177355.257415:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177355.257418:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177355.257422:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:0.0:1786177355.257427:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000001:3.0:1786177355.257427:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.257433:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:0.0:1786177355.257435:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:0.0:1786177355.257440:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.257442:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177355.257443:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177355.257444:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.257448:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177355.257450:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000400:00000200:3.0:1786177355.257453:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177355.257458:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177355.257462:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177355.257464:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1786177355.257467:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302254797
02000000:00000001:0.0:1786177355.257468:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177355.257473:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177355.257477:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000010:3.0:1786177355.257478:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37c60.
00000008:00100000:0.0:1786177355.257480:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177355.257484:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177355.257486:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.257488:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000010:0.0:1786177355.257491:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a680.
00000020:00000040:0.0:1786177355.257495:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
00000100:00000001:3.0:1786177355.257496:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302254797
02000000:00000001:0.0:1786177355.257500:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177355.257503:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177355.257503:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37f60.
00000100:00000001:0.0:1786177355.257506:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177355.257509:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1786177355.257532:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177355.257538:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177355.257541:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.257543:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177355.257548:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177355/real 1786177355]  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177355.257557:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177355.257559:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177355.257561:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177355.257564:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177355.257566:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177355.257570:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177355.257574:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177355.257583:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.257590:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177355.257591:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.257593:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177355.257596:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1786177355.257599:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177355.257601:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1786177355.257605:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000001:0.0:1786177355.257610:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.257617:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1786177355.257618:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177355.257627:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1786177355.257627:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177355.257629:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177355.257636:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000911ee1e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.257651:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1786177355.257652:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177355.257656:0:62275:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1786177355.257657:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1786177355.257662:0:62275:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177355.257664:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1786177355.257665:0:62275:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177355.257668:0:62275:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177355.257668:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177355.257674:0:62275:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1786177355.257675:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00080000:0.0:1786177355.257676:0:62275:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7578
00000100:00000001:3.0:1786177355.257680:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000400:00000200:0.0:1786177355.257682:0:62275:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177355.257686:0:62275:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177355.257688:0:62275:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00080000:3.0:1786177355.257688:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:0.0:1786177355.257693:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.257697:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00080000:3.0:1786177355.257698:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000040:0.0:1786177355.257702:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177355.257704:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1786177355.257706:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177355.257711:0:62275:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000040:3.0:1786177355.257713:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177355.257714:0:62275:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177355.257717:0:62275:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177355.257722:0:62275:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:0.0:1786177355.257726:0:62275:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:0.0:1786177355.257731:0:62275:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00100000:3.0:1786177355.257735:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856368512:192.168.202.110@tcp:250:kworker.0
00000100:00000040:0.0:1786177355.257738:0:62275:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177355.257743:0:62275:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1786177355.257744:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1786177355.257747:0:62275:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177355.257749:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1786177355.257751:0:62275:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000100:00000040:3.0:1786177355.257756:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41ad80 x1872934856368512/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1786177355.257757:0:62275:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177355.257765:0:62275:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177355.257769:0:62275:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177355.257771:0:62275:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177355.257774:0:62275:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177355.257777:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1786177355.257778:0:62275:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177355.257782:0:62275:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177355.257784:0:62275:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177355.257787:0:62275:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177355.257788:0:62275:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177355.257789:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000010:0.0:1786177355.257792:0:62275:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41a300.
02000000:00000010:3.0:1786177355.257794:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
00000020:00000040:0.0:1786177355.257796:0:62275:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177355.257799:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000001:0.0:1786177355.257801:0:62275:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000020:00000001:3.0:1786177355.257803:0:2391:0:(genops.c:1238:class_import_put()) Process entered
02000000:00000001:0.0:1786177355.257804:0:62275:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1786177355.257806:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000100:00000001:0.0:1786177355.257808:0:62275:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177355.257811:0:62275:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000020:00000001:3.0:1786177355.257811:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000001:0.0:1786177355.257815:0:62275:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177355.257816:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7c00.
02000000:00000010:0.0:1786177355.257819:0:62275:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6000.
02000000:00000001:3.0:1786177355.257822:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000001:0.0:1786177355.257823:0:62275:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177355.257828:0:62275:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1786177355.257829:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177355.257834:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41ad80.
00000100:00000001:0.0:1786177355.257835:0:62275:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177355.257838:0:62275:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.257841:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000040:0.0:1786177355.257842:0:62275:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009b0d3569] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.257845:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.257852:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177355.257853:0:62275:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177355.257860:0:62275:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7588)
00000100:00000001:3.0:1786177355.257864:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177355.257867:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.257871:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177355.257872:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177355.257873:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177355.257879:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.257887:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177355.257888:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177355.257891:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177355.257893:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177355.257895:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177355.257898:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177355.257901:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177355.257903:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177355.257905:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177355.257907:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177355.257910:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177355.257913:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177355.257917:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177355.257921:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177355.257925:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177355.257929:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856368640, portal 10
00000100:00000001:3.0:1786177355.257931:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177355.257932:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761665536 : -114547947886080 : ffff97d1ba41aa00)
00000100:00000040:3.0:1786177355.257938:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.257942:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177355.257943:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177355.257945:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177355.257947:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856368640, offset 0
00000400:00000200:3.0:1786177355.257951:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177355.257955:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177355.257959:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177355.257963:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177355.257967:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177355.257970:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177355.257972:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177355.257976:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255051:255051:256:4294967295] 192.168.202.110@tcp LPNI seq info [255040:255040:8:4294967295]
00000400:00000200:3.0:1786177355.257981:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177355.257990:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177355.257995:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177355.257999:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177355.258002:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177355.258004:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786177355.258007:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177355.258009:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1786177355.258011:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177355.258013:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177355.258015:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177355.258018:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177355.258020:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177355.258021:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177355.258024:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177355.258027:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258033:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177355.258035:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258039:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.258041:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177355.258044:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177355.258047:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177355.258049:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177355.258051:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302251018
00000100:00000001:3.0:1786177355.258053:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302254797
00000100:00000001:3.0:1786177355.258054:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302251018
00000100:00000001:3.0:1786177355.258056:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302254797
00000100:00000001:3.0:1786177355.258057:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177355.258058:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.258059:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177355.258061:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177355/real 1786177355]  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177355.258066:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177355.258067:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177355.258069:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177355.258070:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177355.258071:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177355.258072:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177355.258074:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177355.258079:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258082:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177355.258083:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258084:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177355.258084:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177355.258086:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258090:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177355.258092:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258097:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177355.258098:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258099:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177355.258100:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258101:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177355.258102:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177355.258104:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177355.258105:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177355.258106:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177355.258108:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177355.258113:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368640:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177355.258115:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177355.258116:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258120:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177355.258121:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177355.258122:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177355.258123:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177355.258125:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177355.258126:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177355.258128:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177355.258130:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177355.258130:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177355.258131:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177355.258133:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177355.258134:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177355.258137:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177355.258138:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177355.258140:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177355.258142:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856368768, portal 10
00000100:00000001:3.0:1786177355.258143:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177355.258144:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761660160 : -114547947891456 : ffff97d1ba419500)
00000100:00000040:3.0:1786177355.258148:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258152:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177355.258153:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177355.258154:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177355.258156:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856368768, offset 0
00000400:00000200:3.0:1786177355.258158:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177355.258160:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177355.258162:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177355.258164:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177355.258165:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177355.258167:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177355.258169:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177355.258171:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255052:255052:256:4294967295] 192.168.202.110@tcp LPNI seq info [255041:255041:8:4294967295]
00000400:00000200:3.0:1786177355.258173:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177355.258177:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177355.258179:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177355.258181:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177355.258183:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177355.258185:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786177355.258187:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177355.258189:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1786177355.258191:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177355.258192:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177355.258194:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177355.258196:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177355.258198:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177355.258200:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177355.258201:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177355.258203:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258208:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177355.258209:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258213:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.258214:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177355.258216:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177355.258218:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177355.258221:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177355.258223:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302251143
00000100:00000001:3.0:1786177355.258226:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302254797
00000100:00000001:3.0:1786177355.258227:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302251143
00000100:00000001:3.0:1786177355.258230:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302254797
00000100:00000001:3.0:1786177355.258231:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177355.258233:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.258246:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177355.258251:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177355/real 1786177355]  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177355.258260:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177355.258261:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177355.258263:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177355.258265:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177355.258267:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177355.258269:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177355.258272:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177355.258280:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258286:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177355.258287:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258289:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177355.258290:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177355.258294:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258301:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177355.258305:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258311:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177355.258313:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258315:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177355.258316:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258318:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177355.258320:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177355.258322:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177355.258325:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177355.258327:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177355.258332:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177355.258339:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368768:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177355.258342:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177355.258344:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258348:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177355.258349:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177355.258352:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177355.258353:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177355.258354:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177355.258356:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177355.258358:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177355.258360:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177355.258361:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177355.258362:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177355.258364:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177355.258365:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177355.258368:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177355.258370:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177355.258372:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177355.258374:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856368896, portal 4
00000100:00000001:3.0:1786177355.258376:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177355.258377:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761664640 : -114547947886976 : ffff97d1ba41a680)
00000100:00000040:3.0:1786177355.258381:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258385:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177355.258386:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177355.258387:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177355.258389:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856368896, offset 0
00000400:00000200:3.0:1786177355.258391:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177355.258394:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177355.258396:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177355.258398:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177355.258399:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177355.258401:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177355.258402:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177355.258404:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255053:255053:256:4294967295] 192.168.202.110@tcp LPNI seq info [255042:255042:8:4294967295]
00000400:00000200:3.0:1786177355.258407:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177355.258411:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177355.258414:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177355.258416:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177355.258418:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177355.258420:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786177355.258423:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177355.258425:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1786177355.258427:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177355.258428:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177355.258430:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177355.258432:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177355.258434:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177355.258435:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177355.258437:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177355.258439:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258443:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177355.258445:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258449:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.258450:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177355.258452:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177355.258454:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177355.258456:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177355.258458:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302251018
00000100:00000001:3.0:1786177355.258459:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302254797
00000100:00000001:3.0:1786177355.258461:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302251018
00000100:00000001:3.0:1786177355.258462:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302254797
00000100:00000001:3.0:1786177355.258463:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177355.258464:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.258465:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177355.258468:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177355/real 1786177355]  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177355.258473:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177355.258474:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177355.258475:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177355.258476:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177355.258477:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177355.258478:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177355.258480:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177355.258485:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258488:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177355.258489:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258490:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177355.258491:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177355.258493:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258498:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177355.258501:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258507:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177355.258527:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258529:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177355.258530:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258532:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177355.258533:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177355.258535:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177355.258539:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177355.258541:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177355.258545:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177355.258552:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856368896:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177355.258556:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177355.258559:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258564:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177355.258566:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177355.258568:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177355.258570:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177355.258571:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177355.258574:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856369024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177355.258576:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177355.258578:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177355.258579:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177355.258581:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177355.258584:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177355.258586:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177355.258588:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177355.258590:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177355.258593:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177355.258596:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856369024, portal 4
00000100:00000001:3.0:1786177355.258597:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177355.258599:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761663744 : -114547947887872 : ffff97d1ba41a300)
00000100:00000040:3.0:1786177355.258603:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258609:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177355.258611:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177355.258612:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177355.258614:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856369024, offset 0
00000400:00000200:3.0:1786177355.258619:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177355.258622:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177355.258624:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177355.258627:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177355.258629:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177355.258631:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177355.258634:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177355.258636:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255054:255054:256:4294967295] 192.168.202.110@tcp LPNI seq info [255043:255043:8:4294967295]
00000400:00000200:3.0:1786177355.258640:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177355.258645:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177355.258648:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177355.258650:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177355.258654:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177355.258656:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1786177355.258659:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177355.258662:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1786177355.258665:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177355.258667:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177355.258670:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177355.258672:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177355.258674:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177355.258676:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177355.258678:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177355.258681:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258687:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177355.258690:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177380 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258695:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.258697:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177355.258699:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177355.258701:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177355.258704:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177355.258706:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302251027
00000100:00000001:3.0:1786177355.258708:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302254797
00000100:00000001:3.0:1786177355.258710:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302251027
00000100:00000001:3.0:1786177355.258711:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302254797
00000100:00000001:3.0:1786177355.258713:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177355.258714:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.258716:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177355.258719:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177355/real 1786177355]  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177355.258724:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177355.258726:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177355.258727:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177355.258729:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177355.258731:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177355.258732:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177355.258735:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177355.258741:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258746:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177355.258747:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258749:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177355.258750:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177355.258753:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258758:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177355.258761:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258766:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177355.258769:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258770:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177355.258771:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258773:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177355.258774:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177355.258777:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177355.258779:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177355.258781:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177355.258784:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177355.258792:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856369024:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177355.258795:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258796:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177355.258799:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41aa00 x1872934856368640/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258804:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177355.258805:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177355.258807:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:3.0:1786177355.258808:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177355.258810:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177355.258811:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177355.258813:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177355.258815:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4800.
02000000:00000001:3.0:1786177355.258817:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177355.258818:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177355.258819:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41aa00.
00000100:00000001:3.0:1786177355.258821:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177355.258823:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258824:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177355.258827:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419500 x1872934856368768/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258838:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177355.258839:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177355.258840:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:3.0:1786177355.258843:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177355.258844:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177355.258845:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177355.258846:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177355.258847:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4000.
02000000:00000001:3.0:1786177355.258848:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177355.258849:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177355.258850:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419500.
00000100:00000001:3.0:1786177355.258852:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177355.258853:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258854:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177355.258857:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a680 x1872934856368896/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258861:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177355.258862:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177355.258863:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:3.0:1786177355.258865:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177355.258866:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177355.258867:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177355.258868:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177355.258869:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:3.0:1786177355.258871:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177355.258872:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177355.258873:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a680.
00000100:00000001:3.0:1786177355.258874:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177355.258875:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258876:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177355.258879:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41a300 x1872934856369024/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177380 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177355.258884:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177355.258885:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177355.258887:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177355.258889:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177355.258890:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177355.258891:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177355.258893:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177355.258894:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6000.
02000000:00000001:3.0:1786177355.258895:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177355.258896:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177355.258897:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41a300.
00000100:00000001:3.0:1786177355.258899:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177355.258899:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258901:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177355.258904:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177355.258905:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177355.258906:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177355.258908:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177355.833372:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177355.833381:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177355.833388:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177355.833390:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177355.833393:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177355.833395:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177355.834111:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177355.834116:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177355.834119:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177356.280640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177356.280647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177356.280650:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177356.280655:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177356.280659:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177356.280663:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177356.280666:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177356.280668:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177356.280671:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177356.856701:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177356.856711:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177356.856719:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177356.856720:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177356.856723:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177356.856725:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177356.857069:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177356.857074:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177356.857078:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177357.307540:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177357.307548:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177357.307550:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177357.307555:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177357.307559:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177357.307563:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177357.307565:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177357.307568:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177357.307571:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177357.882770:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177357.882779:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177357.882785:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177357.882786:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177357.882789:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177357.882790:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177357.882865:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177357.882871:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177357.882876:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177358.328624:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177358.328630:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177358.328632:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177358.328636:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177358.328639:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177358.328641:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177358.328643:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177358.328645:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177358.328647:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177358.904651:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177358.904659:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177358.904665:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177358.904666:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177358.904669:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177358.904670:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177358.906065:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177358.906069:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177358.906073:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177359.096647:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302254797, at 4302254848
00000100:00000010:1.0:1786177359.096664:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37f60.
00000100:00000040:0.0:1786177359.096749:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302254797, at 4302254848
00000100:00000010:0.0:1786177359.096756:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37c60.
00000100:00000001:3.0:1786177359.352593:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177359.352600:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177359.352603:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177359.352608:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177359.352612:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177359.352615:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177359.352618:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177359.352620:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177359.352633:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177359.928639:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177359.928646:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177359.928652:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177359.928654:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177359.928657:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177359.928658:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177359.928711:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177359.928714:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177359.928717:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177360.378294:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177360.378303:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177360.378305:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177360.378310:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177360.378314:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177360.378318:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.378321:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177360.378324:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.378327:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177360.378461:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177360.378468:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1786177360.378471:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177360.378474:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177360.378477:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177360.378484:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7583
00000400:00000200:1.0:1786177360.378494:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177360.378500:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177360.378503:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177360.378506:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.378527:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177360.378530:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.378533:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177360.378537:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177360.378539:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:1.0:1786177360.378542:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177360.378545:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:1.0:1786177360.378548:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:1.0:1786177360.378550:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:1.0:1786177360.378553:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.378555:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177360.378557:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.378559:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177360.378562:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:1.0:1786177360.378566:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177360.378568:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177360.378572:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177360.378575:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177360.378578:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.378580:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:1.0:1786177360.378582:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177360.378591:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa4000.
00000020:00000040:1.0:1786177360.378594:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:1.0:1786177360.378598:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177360.378600:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.378603:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177360.378606:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177360.378610:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177360.378618:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b02c00.
00000100:00000001:1.0:1786177360.378624:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177360.378634:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177360.378644:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177360.378648:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177360.378652:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd947f83] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.378712:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177360.378715:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177360.378717:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177360.378723:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.378730:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177360.378732:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177360.378735:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177360.378739:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177360.378742:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177360.378745:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856369152:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177360.378748:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177360.378752:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177360.378755:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177360.378758:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177360.378763:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177360.378765:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177360.378770:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177360.378774:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177360.378781:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177360.378787:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856369152, portal 25
00000100:00000001:3.0:1786177360.378790:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177360.378791:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751742464 : -114547957809152 : ffff97d1b9aa4000)
00000100:00000040:3.0:1786177360.378797:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.378804:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177360.378807:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177360.378809:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177360.378812:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856369152, offset 0
00000400:00000200:3.0:1786177360.378818:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177360.378825:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177360.378831:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177360.378836:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177360.378840:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177360.378844:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177360.378846:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177360.378850:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255055:255055:256:4294967295] 192.168.202.110@tcp LPNI seq info [255044:255044:8:4294967295]
00000400:00000200:3.0:1786177360.378855:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177360.378865:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177360.378871:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177360.378876:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177360.378882:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177360.378886:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786177360.378903:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177360.378908:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1786177360.378912:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177360.378915:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177360.378921:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177360.378925:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177360.378931:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177360.378933:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177360.378937:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177360.378940:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177360.379009:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177360.379013:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177360.379016:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1786177360.379018:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177360.379019:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177360.379022:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177360.379025:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7583
00000400:00000200:1.0:1786177360.379029:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177360.379032:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177360.379034:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177360.379036:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.379038:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177360.379040:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.379042:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177360.379044:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177360.379046:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:1.0:1786177360.379048:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177360.379050:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:1.0:1786177360.379052:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:1.0:1786177360.379054:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:1.0:1786177360.379056:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.379058:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177360.379060:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.379062:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177360.379065:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:1.0:1786177360.379067:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177360.379069:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177360.379072:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177360.379076:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177360.379078:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.379080:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177360.379084:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1786177360.379086:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.379088:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177360.379092:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa7480.
00000020:00000040:1.0:1786177360.379095:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786177360.379097:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177360.379098:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.379099:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177360.379101:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177360.379103:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177360.379106:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b00c00.
00000100:00000001:1.0:1786177360.379110:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177360.379114:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177360.379122:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177360.379124:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177360.379128:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000492901e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379207:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177360.379212:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379220:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.379222:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177360.379229:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177360.379233:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177360.379236:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302259919
00000100:00000010:3.0:1786177360.379241:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad374e0.
00000100:00000001:3.0:1786177360.379247:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:3.0:1786177360.379251:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302259919
00000100:00000010:3.0:1786177360.379254:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1aad37120.
00000100:00000001:3.0:1786177360.379257:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:3.0:1786177360.379260:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177360.379263:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.379265:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177360.379269:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177360/real 1786177360]  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177360.379284:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177360.379285:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177360.379287:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177360.379290:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177360.379292:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177360.379295:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177360.379298:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177360.379307:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379314:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177360.379315:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.379318:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177360.379320:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177360.379336:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379343:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177360.379348:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379355:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177360.379358:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.379359:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177360.379361:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.379363:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177360.379365:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177360.379368:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177360.379371:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177360.379373:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177360.379378:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177360.379386:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856369152:192.168.202.110@tcp:250:kworker.0
00000100:00000001:3.0:1786177360.379390:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.379392:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177360.379396:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4000 x1872934856369152/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379402:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177360.379405:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177360.379407:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177360.379410:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177360.379411:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177360.379413:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:3.0:1786177360.379415:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177360.379417:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b02c00.
02000000:00000001:3.0:1786177360.379422:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177360.379423:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177360.379425:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4000.
00000100:00000001:3.0:1786177360.379428:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177360.379429:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.379431:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.379439:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177360.379440:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.379442:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177360.379443:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177360.379445:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177360.379448:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379455:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177360.379456:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177360.379458:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177360.379459:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177360.379461:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177360.379464:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856369280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177360.379468:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177360.379470:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177360.379471:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177360.379473:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177360.379476:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177360.379478:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177360.379481:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177360.379484:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177360.379487:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177360.379490:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856369280, portal 10
00000100:00000001:3.0:1786177360.379492:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177360.379493:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751755904 : -114547957795712 : ffff97d1b9aa7480)
00000100:00000040:3.0:1786177360.379498:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379505:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177360.379507:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177360.379509:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177360.379544:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856369280, offset 0
00000400:00000200:3.0:1786177360.379549:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177360.379553:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177360.379556:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177360.379559:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177360.379562:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177360.379564:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177360.379566:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177360.379569:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255056:255056:256:4294967295] 192.168.202.110@tcp LPNI seq info [255045:255045:8:4294967295]
00000400:00000200:3.0:1786177360.379573:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177360.379580:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177360.379583:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177360.379586:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177360.379590:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177360.379593:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786177360.379597:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177360.379600:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1786177360.379604:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177360.379606:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177360.379609:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177360.379613:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177360.379616:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177360.379618:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177360.379621:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177360.379624:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379632:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177360.379635:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379641:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.379643:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177360.379646:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177360.379649:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:3.0:1786177360.379652:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177360.379654:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302256139
00000100:00000001:3.0:1786177360.379657:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302259919
00000100:00000001:3.0:1786177360.379659:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302256139
00000100:00000001:3.0:1786177360.379661:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302259919
00000100:00000001:3.0:1786177360.379663:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177360.379665:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.379666:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177360.379670:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177360/real 1786177360]  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177360.379689:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177360.379691:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177360.379693:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177360.379696:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177360.379698:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177360.379700:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177360.379704:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177360.379712:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379718:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177360.379720:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.379721:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177360.379722:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177360.379726:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379733:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177360.379736:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379742:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177360.379745:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.379746:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177360.379747:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.379749:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177360.379751:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177360.379754:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177360.379757:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177360.379759:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177360.379763:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177360.379770:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856369280:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177360.379774:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.379776:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177360.379779:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa7480 x1872934856369280/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.379785:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177360.379787:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177360.379788:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177360.379790:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177360.379792:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177360.379793:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177360.379795:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177360.379796:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00c00.
02000000:00000001:3.0:1786177360.379798:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177360.379800:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177360.379801:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa7480.
00000100:00000001:3.0:1786177360.379803:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177360.379805:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.379806:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.379810:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177360.379811:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.379813:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177360.379814:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.380523:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177360.380525:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.380564:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177360.380568:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177360.380572:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1786177360.380573:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177360.380575:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177360.380578:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177360.380582:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7583
00000400:00000200:1.0:1786177360.380586:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177360.380590:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177360.380592:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177360.380595:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.380597:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177360.380599:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.380601:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177360.380603:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177360.380604:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:1.0:1786177360.380606:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177360.380608:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:1.0:1786177360.380610:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:1.0:1786177360.380612:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:1.0:1786177360.380615:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.380617:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177360.380619:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.380621:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177360.380623:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:1.0:1786177360.380626:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177360.380628:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177360.380630:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177360.380634:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177360.380636:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.380638:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177360.380641:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1786177360.380643:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.380645:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177360.380650:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa7480.
00000020:00000040:1.0:1786177360.380653:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:1.0:1786177360.380656:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177360.380657:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.380659:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177360.380661:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177360.380662:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177360.380666:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b00800.
00000100:00000001:1.0:1786177360.380670:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177360.380736:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177360.380745:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177360.380747:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177360.380751:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000492901e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.380791:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177360.380793:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177360.380794:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177360.380799:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.380806:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177360.380807:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177360.380809:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177360.380811:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177360.380813:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177360.380816:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856369408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177360.380820:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177360.380821:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177360.380823:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177360.380825:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177360.380829:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177360.380831:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177360.380834:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177360.380837:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177360.380841:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177360.380845:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856369408, portal 10
00000100:00000001:3.0:1786177360.380847:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177360.380848:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751755904 : -114547957795712 : ffff97d1b9aa7480)
00000100:00000040:3.0:1786177360.380853:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.380860:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177360.380862:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177360.380864:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177360.380867:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856369408, offset 0
00000400:00000200:3.0:1786177360.380871:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177360.380875:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177360.380879:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177360.380882:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177360.380885:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177360.380888:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177360.380948:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177360.380952:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255057:255057:256:4294967295] 192.168.202.110@tcp LPNI seq info [255046:255046:8:4294967295]
00000400:00000200:3.0:1786177360.380957:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177360.380963:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177360.380968:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177360.380972:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177360.380976:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177360.380979:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786177360.380983:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177360.380986:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1786177360.380990:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177360.380992:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177360.380995:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177360.380999:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177360.381002:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177360.381004:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177360.381007:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177360.381010:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177360.381604:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177360.381608:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177360.381611:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:1.0:1786177360.381612:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177360.381614:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177360.381617:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177360.381620:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7583
00000400:00000200:1.0:1786177360.381624:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177360.381627:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177360.381630:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177360.381633:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.381635:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177360.381637:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.381639:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177360.381642:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177360.381643:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:1.0:1786177360.381645:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177360.381648:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:1.0:1786177360.381649:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:1.0:1786177360.381651:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:1.0:1786177360.381654:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.381656:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177360.381658:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.381660:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177360.381663:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:1.0:1786177360.381666:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177360.381668:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177360.381669:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177360.381672:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177360.381685:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.381687:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177360.381689:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1786177360.381692:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.381694:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177360.381698:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa4000.
00000020:00000040:1.0:1786177360.381701:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:1.0:1786177360.381704:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177360.381705:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.381707:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177360.381708:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177360.381711:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177360.381714:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b03400.
00000100:00000001:1.0:1786177360.381717:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177360.381723:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177360.381730:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177360.381733:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177360.381737:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bd947f83] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.381837:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177360.381843:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.381850:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.381852:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177360.381858:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177360.381862:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:3.0:1786177360.381865:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177360.381867:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302256267
00000100:00000001:3.0:1786177360.381870:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302259919
00000100:00000001:3.0:1786177360.381872:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302256267
00000100:00000001:3.0:1786177360.381874:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302259919
00000100:00000001:3.0:1786177360.381877:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177360.381879:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.381881:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177360.381884:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177360/real 1786177360]  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177360.381906:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177360.381907:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177360.381910:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177360.381913:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177360.381915:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177360.381917:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177360.381921:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177360.381930:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.381937:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177360.381939:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.381941:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177360.381943:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177360.381947:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.381953:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177360.381957:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.381964:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177360.381967:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.381968:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177360.381969:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.381971:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177360.381973:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177360.381975:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177360.381979:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177360.381981:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177360.381984:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177360.381992:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856369408:192.168.202.110@tcp:38:kworker.0
00000100:00000001:3.0:1786177360.381997:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.381999:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177360.382003:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa7480 x1872934856369408/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.382010:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177360.382012:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177360.382014:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177360.382017:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177360.382018:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177360.382020:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:3.0:1786177360.382022:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177360.382024:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00800.
02000000:00000001:3.0:1786177360.382027:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177360.382028:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177360.382030:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa7480.
00000100:00000001:3.0:1786177360.382032:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177360.382033:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.382035:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.382040:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177360.382042:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.382044:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177360.382046:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177360.382047:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177360.382050:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.382057:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177360.382059:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177360.382061:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177360.382062:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177360.382064:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177360.382067:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856369536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177360.382071:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177360.382073:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177360.382075:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177360.382077:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177360.382080:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177360.382082:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177360.382085:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177360.382088:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177360.382091:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177360.382094:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856369536, portal 4
00000100:00000001:3.0:1786177360.382096:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177360.382097:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751742464 : -114547957809152 : ffff97d1b9aa4000)
00000100:00000040:3.0:1786177360.382102:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.382109:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177360.382111:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177360.382113:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177360.382115:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856369536, offset 0
00000400:00000200:3.0:1786177360.382119:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177360.382123:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177360.382127:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177360.382130:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177360.382133:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177360.382136:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177360.382139:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177360.382142:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255058:255058:256:4294967295] 192.168.202.110@tcp LPNI seq info [255047:255047:8:4294967295]
00000400:00000200:3.0:1786177360.382146:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177360.382154:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177360.382159:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177360.382162:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177360.382167:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177360.382169:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786177360.382173:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177360.382176:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1786177360.382180:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177360.382182:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177360.382264:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177360.382270:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177360.382273:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177360.382275:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177360.382278:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177360.382282:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.382290:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177360.382293:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.382299:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.382301:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177360.382304:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177360.382307:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:3.0:1786177360.382311:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177360.382312:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302256142
00000100:00000001:3.0:1786177360.382315:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302259919
00000100:00000001:3.0:1786177360.382317:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302256142
00000100:00000001:3.0:1786177360.382319:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302259919
00000100:00000001:3.0:1786177360.382322:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177360.382381:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.382383:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177360.382387:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177360/real 1786177360]  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177360.382395:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177360.382397:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177360.382399:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177360.382401:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177360.382403:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177360.382405:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177360.382408:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177360.382417:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.382423:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177360.382424:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.382426:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177360.382427:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177360.382430:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.382437:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177360.382441:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.382447:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177360.382449:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.382451:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177360.382452:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.382454:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177360.382455:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177360.382458:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177360.382461:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177360.382464:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177360.382467:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177360.382475:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856369536:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177360.382478:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.382480:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177360.382484:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa4000 x1872934856369536/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.382490:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177360.382492:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177360.382493:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177360.382496:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177360.382497:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177360.382498:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177360.382500:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177360.382502:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b03400.
02000000:00000001:3.0:1786177360.382504:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177360.382506:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177360.382507:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa4000.
00000100:00000001:3.0:1786177360.382524:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177360.382526:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.382528:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.382532:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177360.382533:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.382535:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177360.382537:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.382638:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177360.382640:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.382726:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1786177360.382731:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1786177360.382734:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1786177360.382735:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1786177360.382737:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1786177360.382740:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1786177360.382743:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7583
00000400:00000200:1.0:1786177360.382747:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177360.382750:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1786177360.382753:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1786177360.382755:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.382758:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177360.382759:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.382761:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:1.0:1786177360.382764:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:1.0:1786177360.382765:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:1.0:1786177360.382766:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:1.0:1786177360.382769:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:1.0:1786177360.382771:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:1.0:1786177360.382773:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:1.0:1786177360.382776:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.382778:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1786177360.382780:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177360.382782:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:1.0:1786177360.382784:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:1.0:1786177360.382786:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:1.0:1786177360.382789:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177360.382790:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1786177360.382793:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177360.382795:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.382796:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:1.0:1786177360.382798:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1786177360.382800:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.382802:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1786177360.382806:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1b9aa6680.
00000020:00000040:1.0:1786177360.382809:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:1.0:1786177360.382811:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1786177360.382812:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177360.382814:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1786177360.382816:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1786177360.382818:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1786177360.382822:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186b00000.
00000100:00000001:1.0:1786177360.382826:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177360.382831:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1786177360.382837:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1786177360.382840:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177360.382843:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dced6303] to pc [ptlrpcd_rcv+-1]  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.382884:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177360.382886:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177360.382887:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1786177360.382947:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.382954:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177360.382955:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177360.382958:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1786177360.382959:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177360.382962:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177360.382965:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1b9aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856369664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177360.382968:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1786177360.382970:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1786177360.382972:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177360.382973:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1786177360.382977:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177360.382978:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1786177360.382982:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:3.0:1786177360.382985:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1ba580898.
00000400:00000010:3.0:1786177360.382988:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514ef68.
00000100:00000200:3.0:1786177360.382991:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856369664, portal 4
00000100:00000001:3.0:1786177360.382993:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1786177360.382994:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525751752320 : -114547957799296 : ffff97d1b9aa6680)
00000100:00000040:3.0:1786177360.382999:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.383005:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1786177360.383008:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:3.0:1786177360.383010:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d18514e4c8.
00000100:00000200:3.0:1786177360.383012:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856369664, offset 0
00000400:00000200:3.0:1786177360.383016:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:3.0:1786177360.383020:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177360.383023:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:3.0:1786177360.383026:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1786177360.383029:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:3.0:1786177360.383031:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1786177360.383033:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:3.0:1786177360.383036:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255059:255059:256:4294967295] 192.168.202.110@tcp LPNI seq info [255048:255048:8:4294967295]
00000400:00000200:3.0:1786177360.383040:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177360.383046:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1786177360.383051:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177360.383054:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1835cd500.
00000800:00000200:3.0:1786177360.383058:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.0:1786177360.383061:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1786177360.383066:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:3.1:1786177360.383069:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1786177360.383072:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:3.0:1786177360.383075:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1835cd500.
00000400:00000200:3.0:1786177360.383077:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1786177360.383080:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1786177360.383083:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514e4c8
00000400:00000010:3.0:1786177360.383085:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514e4c8.
00000100:00000001:3.0:1786177360.383087:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1786177360.383090:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177360.383112:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177360.383118:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7593)
00000100:00000001:3.0:1786177360.383153:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177360.383157:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.383164:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.383165:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1786177360.383169:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177360.383172:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:3.0:1786177360.383175:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:3.0:1786177360.383177:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302256151
00000100:00000001:3.0:1786177360.383180:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302259919
00000100:00000001:3.0:1786177360.383182:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302256151
00000100:00000001:3.0:1786177360.383184:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302259919
00000100:00000001:3.0:1786177360.383197:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1786177360.383200:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.383201:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1786177360.383205:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177360/real 1786177360]  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1786177360.383213:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1786177360.383214:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1786177360.383216:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1ba580898.
00000400:00000200:3.0:1786177360.383219:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d18514ef68
00000400:00000010:3.0:1786177360.383220:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d18514ef68.
00000100:00000001:3.0:1786177360.383223:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1786177360.383227:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1786177360.383234:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.383241:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1786177360.383242:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.383244:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177360.383246:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1786177360.383249:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.383256:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1786177360.383260:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.383267:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1786177360.383269:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.383271:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1786177360.383272:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.383273:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1786177360.383275:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1786177360.383278:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1786177360.383281:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:3.0:1786177360.383283:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1786177360.383287:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1786177360.383296:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1b9aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856369664:192.168.202.110@tcp:8:kworker.0
00000100:00000001:3.0:1786177360.383299:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.383301:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1786177360.383304:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1b9aa6680 x1872934856369664/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177360.383310:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1786177360.383312:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1786177360.383313:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:3.0:1786177360.383316:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1786177360.383317:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:3.0:1786177360.383319:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:3.0:1786177360.383320:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:3.0:1786177360.383383:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186b00000.
02000000:00000001:3.0:1786177360.383387:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1786177360.383388:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1786177360.383390:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1b9aa6680.
00000100:00000001:3.0:1786177360.383392:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1786177360.383394:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.383395:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177360.383399:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177360.383400:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177360.383402:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1786177360.383403:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177360.952806:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177360.952813:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177360.952830:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177360.952832:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177360.952835:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177360.952837:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177360.952972:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177360.952976:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177360.952979:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177361.400649:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177361.400657:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177361.400660:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177361.400666:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177361.400670:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177361.400674:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177361.400677:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177361.400680:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177361.400683:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177361.977313:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177361.977322:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177361.977324:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177361.977328:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177361.977333:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177361.977336:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177361.977338:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177361.977340:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177361.977344:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177361.977346:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177361.977347:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177361.977349:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177361.977351:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177361.977353:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177361.977354:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177361.977356:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177361.977360:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177361.977362:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177361.977467:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177361.977471:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177361.977473:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177362.425874:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177362.425881:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177362.425884:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177362.425889:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177362.425893:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177362.425896:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177362.425898:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177362.425901:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177362.425904:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177363.002151:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177363.002158:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177363.002191:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177363.002193:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177363.002196:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177363.002197:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177363.002347:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177363.002356:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177363.002359:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177363.449088:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177363.449094:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177363.449096:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177363.449100:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177363.449103:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177363.449106:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177363.449109:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177363.449110:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177363.449113:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177364.024654:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177364.024666:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177364.024672:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177364.024674:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177364.024677:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177364.024677:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177364.024777:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177364.024781:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177364.024784:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177364.216607:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302259919, at 4302259968
00000100:00000040:1.0:1786177364.216607:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302259919, at 4302259968
00000100:00000010:0.0:1786177364.216623:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad374e0.
00000100:00000010:1.0:1786177364.216625:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1aad37120.
00000100:00000001:3.0:1786177364.473279:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177364.473286:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177364.473289:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177364.473294:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177364.473297:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177364.473301:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177364.473304:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177364.473307:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177364.473310:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177365.048628:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177365.048636:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177365.048641:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177365.048643:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177365.048646:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177365.048649:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177365.048710:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177365.048715:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177365.048717:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177365.048720:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177365.048722:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177365.048724:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177365.048899:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177365.048903:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177365.048906:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786177365.496688:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177365.496701:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177365.496704:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177365.496707:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177365.496712:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177365.496719:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7588
00000400:00000200:3.0:1786177365.496728:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177365.496828:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177365.496832:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177365.496835:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.496838:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177365.496840:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.496842:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177365.496847:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177365.496849:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177365.496851:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177365.496854:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177365.496856:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177365.496859:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177365.496861:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.496863:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177365.496865:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.496867:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177365.496870:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177365.496873:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177365.496875:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177365.496878:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177365.496881:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177365.496883:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.496885:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177365.496888:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177365.496897:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69dc00.
00000020:00000040:3.0:1786177365.496900:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177365.496904:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177365.496906:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.496909:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177365.496912:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177365.496915:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177365.496922:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000001:3.0:1786177365.496929:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177365.496937:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177365.496947:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177365.496951:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177365.496955:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097fe80e7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.497025:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177365.497029:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177365.497032:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177365.497038:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.497051:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177365.497053:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177365.497056:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177365.497059:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177365.497061:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177365.497064:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856369792:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177365.497068:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177365.497072:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177365.497075:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177365.497077:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177365.497082:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177365.497084:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177365.497089:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000400:00000200:2.0:1786177365.497090:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177365.497095:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000400:00000010:2.0:1786177365.497097:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000100:00000001:0.0:1786177365.497098:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177365.497104:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000400:00000010:2.0:1786177365.497106:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177365.497115:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856369792, portal 25
00000100:00000001:2.0:1786177365.497117:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177365.497119:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764299776 : -114547945251840 : ffff97d1ba69dc00)
00000100:00000040:2.0:1786177365.497125:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.497132:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177365.497135:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497156:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177365.497159:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177365.497163:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177365.497164:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177365.497166:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177365.497169:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177365.497172:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7588
00000100:00000001:0.0:1786177365.497176:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000400:00000200:3.0:1786177365.497180:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177365.497181:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177365.497185:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000400:00000200:3.0:1786177365.497186:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1786177365.497189:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1786177365.497190:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1786177365.497194:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177365.497195:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497199:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177365.497202:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497204:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177365.497206:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177365.497207:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177365.497209:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177365.497211:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786177365.497213:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786177365.497216:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177365.497219:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497221:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177365.497223:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497224:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177365.497226:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786177365.497229:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497231:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177365.497233:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177365.497238:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177365.497240:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.497242:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177365.497246:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177365.497248:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.497251:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177365.497256:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e680.
00000020:00000040:3.0:1786177365.497259:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177365.497261:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177365.497262:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.497263:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177365.497265:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177365.497267:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177365.497270:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:3.0:1786177365.497274:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177365.497279:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177365.497286:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177365.497288:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177365.497291:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1839131] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177365.497299:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177365.497301:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177365.497304:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177365.497305:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177365.497307:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177365.497309:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177365.497311:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7588
00000400:00000200:3.0:1786177365.497314:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177365.497317:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177365.497319:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177365.497322:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497324:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177365.497325:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497327:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177365.497329:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177365.497330:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177365.497332:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177365.497333:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786177365.497335:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177365.497337:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177365.497339:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497341:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177365.497343:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497345:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177365.497346:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786177365.497349:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497351:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177365.497352:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177365.497355:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177365.497357:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.497359:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177365.497361:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177365.497363:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.497364:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177365.497368:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ed80.
00000020:00000040:3.0:1786177365.497370:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177365.497373:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177365.497374:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1786177365.497407:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177365.497412:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856369792, offset 0
00000400:00000200:2.0:1786177365.497419:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177365.497437:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177365.497443:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177365.497447:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177365.497451:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177365.497455:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177365.497458:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177365.497462:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255060:255060:256:4294967295] 192.168.202.110@tcp LPNI seq info [255049:255049:8:4294967295]
00000400:00000200:2.0:1786177365.497466:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177365.497475:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177365.497481:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177365.497486:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177365.497493:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177365.497496:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1786177365.497502:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177365.497505:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1786177365.497509:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177365.497536:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177365.497543:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177365.497547:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177365.497552:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177365.497554:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177365.497559:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1786177365.497860:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177365.497863:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177365.497865:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177365.497870:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:3.0:1786177365.497875:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177365.497880:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177365.497887:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177365.497889:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177365.497892:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ccd6c6d2] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177365.497899:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177365.497902:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177365.497905:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177365.497906:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177365.497907:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177365.497910:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177365.497912:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7588
00000400:00000200:3.0:1786177365.497916:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177365.497919:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177365.497922:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177365.497924:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497926:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177365.497928:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497930:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177365.497932:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177365.497933:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177365.497935:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177365.497937:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177365.497939:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177365.497941:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177365.497944:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497945:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177365.497948:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497949:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177365.497951:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177365.497954:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177365.497956:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177365.497957:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177365.497960:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177365.497962:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.497963:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177365.497966:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177365.497968:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.497970:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177365.497974:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c380.
00000020:00000040:3.0:1786177365.497976:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177365.497978:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177365.497979:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.497981:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177365.497982:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177365.497984:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177365.497988:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:3.0:1786177365.497991:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177365.497995:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177365.498002:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177365.498004:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177365.498008:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000833bf1c9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177365.498014:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177365.498017:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177365.498020:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177365.498021:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177365.498023:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177365.498026:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177365.498028:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7588
00000400:00000200:3.0:1786177365.498031:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177365.498034:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177365.498036:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177365.498038:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.498040:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177365.498042:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.498045:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177365.498047:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177365.498048:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177365.498050:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177365.498052:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177365.498053:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177365.498055:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177365.498057:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.498059:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177365.498061:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177365.498063:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177365.498065:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786177365.498067:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177365.498070:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177365.498071:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177365.498074:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177365.498075:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.498077:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177365.498079:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177365.498081:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.498083:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177365.498087:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d880.
00000020:00000040:3.0:1786177365.498089:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177365.498091:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177365.498092:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177365.498094:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177365.498096:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177365.498097:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177365.498101:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:3.0:1786177365.498103:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177365.498108:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177365.498114:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177365.498116:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177365.498119:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097f9b8db] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177365.498126:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177365.498130:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7598)
00000100:00000200:2.0:1786177365.498160:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.498170:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177365.498173:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.498180:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177365.498182:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177365.498189:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177365.498193:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177365.498196:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302265038
00000100:00000010:2.0:1786177365.498201:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25ea0.
00000100:00000001:2.0:1786177365.498207:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177365.498213:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302265038
00000100:00000010:2.0:1786177365.498216:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25960.
00000100:00000001:2.0:1786177365.498218:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786177365.498221:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177365.498224:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177365.498227:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177365.498231:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177365/real 1786177365]  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177365.498239:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177365.498241:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177365.498243:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177365.498246:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177365.498248:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177365.498251:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177365.498254:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177365.498262:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.498269:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177365.498270:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.498272:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177365.498274:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177365.498278:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.498284:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177365.498288:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.498295:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177365.498298:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.498299:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177365.498300:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.498303:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177365.498304:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177365.498307:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177365.498311:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177365.498313:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177365.498317:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177365.498326:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856369792:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177365.498330:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.498332:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177365.498335:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69dc00 x1872934856369792/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.498342:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177365.498344:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177365.498346:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177365.498349:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177365.498351:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177365.498352:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1786177365.498355:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177365.498357:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:2.0:1786177365.499080:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177365.499083:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177365.499085:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69dc00.
00000100:00000001:2.0:1786177365.499089:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177365.499090:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.499093:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.499099:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177365.499101:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177365.499104:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177365.499106:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177365.499107:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177365.499112:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.499120:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177365.499121:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177365.499124:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177365.499126:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177365.499128:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177365.499131:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856369920:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177365.499135:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177365.499206:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177365.499209:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177365.499211:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177365.499215:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177365.499217:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177365.499221:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177365.499224:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177365.499228:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177365.499232:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856369920, portal 10
00000100:00000001:2.0:1786177365.499233:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177365.499235:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764302464 : -114547945249152 : ffff97d1ba69e680)
00000100:00000040:2.0:1786177365.499241:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.499247:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177365.499249:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177365.499251:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177365.499254:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856369920, offset 0
00000400:00000200:2.0:1786177365.499258:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177365.499263:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177365.499267:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177365.499270:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177365.499273:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177365.499276:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177365.499279:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177365.499282:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255061:255061:256:4294967295] 192.168.202.110@tcp LPNI seq info [255050:255050:8:4294967295]
00000400:00000200:2.0:1786177365.499286:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177365.499293:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177365.499297:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177365.499301:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177365.499306:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177365.499308:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1786177365.499313:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177365.499316:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1786177365.499319:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177365.499321:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177365.499325:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177365.499328:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177365.499332:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177365.499333:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177365.499336:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177365.499340:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.499348:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177365.499350:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.499356:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177365.499358:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177365.499361:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177365.499364:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:2.0:1786177365.499368:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177365.499369:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302261259
00000100:00000001:2.0:1786177365.499372:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302265038
00000100:00000001:2.0:1786177365.499374:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302261259
00000100:00000001:2.0:1786177365.499376:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302265038
00000100:00000001:2.0:1786177365.499378:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177365.499477:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177365.499481:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177365.499486:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177365/real 1786177365]  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177365.499501:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177365.499503:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177365.499505:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177365.499508:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177365.499509:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177365.499526:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177365.499530:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177365.499538:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.499544:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177365.499546:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.499547:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177365.499549:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177365.499552:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.499558:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177365.499663:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.499672:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177365.499675:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.499677:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177365.499678:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.499680:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177365.499682:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177365.499684:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177365.499688:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177365.499690:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177365.499695:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177365.499703:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856369920:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177365.499707:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177365.499710:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.499716:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177365.499718:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177365.499720:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177365.499722:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177365.499723:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177365.499726:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177365.499729:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177365.499789:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177365.499792:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177365.499794:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177365.499798:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177365.499800:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177365.499804:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177365.499806:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177365.499810:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177365.499813:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856370048, portal 10
00000100:00000001:2.0:1786177365.499814:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177365.499816:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764304256 : -114547945247360 : ffff97d1ba69ed80)
00000100:00000040:2.0:1786177365.499822:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.499829:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177365.499832:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177365.499834:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177365.499836:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856370048, offset 0
00000400:00000200:2.0:1786177365.499841:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177365.499845:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177365.499848:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177365.499851:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177365.499854:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177365.499856:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177365.499858:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177365.499861:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255062:255062:256:4294967295] 192.168.202.110@tcp LPNI seq info [255051:255051:8:4294967295]
00000400:00000200:2.0:1786177365.499865:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177365.499871:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177365.499876:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177365.499879:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177365.499883:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177365.499886:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1786177365.499890:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177365.499893:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1786177365.499897:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177365.499899:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177365.499902:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177365.499905:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177365.499908:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177365.499909:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177365.499912:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177365.499915:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.499923:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177365.499926:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.499931:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177365.499933:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177365.499936:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177365.499938:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:2.0:1786177365.499941:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177365.499944:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302261385
00000100:00000001:2.0:1786177365.499946:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302265038
00000100:00000001:2.0:1786177365.499949:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302261385
00000100:00000001:2.0:1786177365.499951:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302265038
00000100:00000001:2.0:1786177365.499953:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177365.499955:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177365.499957:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177365.499961:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177365/real 1786177365]  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177365.499968:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177365.499969:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177365.499971:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177365.499973:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177365.499974:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177365.499976:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177365.499979:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177365.499987:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.499993:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177365.499994:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.499996:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177365.499997:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177365.500000:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500006:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177365.500010:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500022:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177365.500025:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.500026:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177365.500028:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.500029:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177365.500031:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177365.500033:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177365.500036:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177365.500039:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177365.500043:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177365.500050:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370048:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177365.500054:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177365.500058:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500064:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177365.500066:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177365.500068:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177365.500070:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177365.500072:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177365.500074:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177365.500077:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177365.500080:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177365.500081:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177365.500083:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177365.500086:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab0000.
02000000:00000001:2.0:1786177365.500088:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177365.500091:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177365.500094:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177365.500097:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177365.500099:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856370176, portal 4
00000100:00000001:2.0:1786177365.500102:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177365.500103:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764293504 : -114547945258112 : ffff97d1ba69c380)
00000100:00000040:2.0:1786177365.500108:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500115:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177365.500118:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177365.500120:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177365.500122:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856370176, offset 0
00000400:00000200:2.0:1786177365.500126:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177365.500130:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177365.500133:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177365.500148:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177365.500151:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177365.500154:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177365.500156:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177365.500159:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255063:255063:256:4294967295] 192.168.202.110@tcp LPNI seq info [255052:255052:8:4294967295]
00000400:00000200:2.0:1786177365.500163:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177365.500169:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177365.500173:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177365.500177:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177365.500180:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177365.500183:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1786177365.500186:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177365.500189:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1786177365.500192:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177365.500194:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177365.500197:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177365.500200:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177365.500203:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177365.500204:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177365.500207:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177365.500211:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500222:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177365.500225:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500230:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177365.500231:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177365.500234:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177365.500237:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:2.0:1786177365.500240:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177365.500242:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302261260
00000100:00000001:2.0:1786177365.500244:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302265038
00000100:00000001:2.0:1786177365.500245:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302261260
00000100:00000001:2.0:1786177365.500247:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302265038
00000100:00000001:2.0:1786177365.500249:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177365.500251:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177365.500252:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177365.500256:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177365/real 1786177365]  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177365.500262:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177365.500263:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177365.500265:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177365.500267:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177365.500268:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177365.500270:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177365.500273:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177365.500280:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500285:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177365.500287:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.500289:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177365.500290:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177365.500293:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500299:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177365.500302:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500308:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177365.500310:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.500312:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177365.500313:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.500314:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177365.500316:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177365.500318:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177365.500321:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177365.500324:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177365.500327:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177365.500335:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370176:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177365.500339:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177365.500342:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500348:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177365.500349:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177365.500351:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177365.500353:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177365.500355:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177365.500358:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177365.500360:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177365.500362:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177365.500364:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177365.500366:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177365.500369:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab1800.
02000000:00000001:2.0:1786177365.500371:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177365.500374:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177365.500481:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177365.500487:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177365.500490:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856370304, portal 4
00000100:00000001:2.0:1786177365.500492:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177365.500493:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764298880 : -114547945252736 : ffff97d1ba69d880)
00000100:00000040:2.0:1786177365.500500:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500505:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177365.500508:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177365.500529:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177365.500533:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856370304, offset 0
00000400:00000200:2.0:1786177365.500538:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177365.500542:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177365.500546:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177365.500548:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177365.500550:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177365.500553:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177365.500555:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177365.500557:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255064:255064:256:4294967295] 192.168.202.110@tcp LPNI seq info [255053:255053:8:4294967295]
00000400:00000200:2.0:1786177365.500675:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177365.500684:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177365.500689:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177365.500692:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177365.500696:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177365.500699:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1786177365.500702:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177365.500705:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1786177365.500708:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177365.500710:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177365.500713:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177365.500716:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177365.500719:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177365.500721:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177365.500723:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177365.500727:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500792:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177365.500796:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500802:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177365.500804:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177365.500807:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177365.500810:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:2.0:1786177365.500814:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177365.500816:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302261269
00000100:00000001:2.0:1786177365.500818:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302265038
00000100:00000001:2.0:1786177365.500820:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302261269
00000100:00000001:2.0:1786177365.500822:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302265038
00000100:00000001:2.0:1786177365.500824:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177365.500826:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177365.500828:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177365.500831:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177365/real 1786177365]  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177365.500839:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177365.500841:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177365.500843:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177365.500845:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177365.500846:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177365.500848:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177365.500851:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177365.500858:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500863:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177365.500865:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.500866:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177365.500867:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177365.500871:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500877:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177365.500881:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500886:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177365.500888:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.500890:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177365.500891:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.500893:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177365.500894:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177365.500897:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177365.500900:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177365.500902:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177365.500906:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177365.500917:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370304:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177365.500920:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.500922:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177365.500925:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e680 x1872934856369920/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500930:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177365.500932:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177365.500934:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177365.500937:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177365.500938:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177365.500940:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177365.500942:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177365.500944:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:2.0:1786177365.500946:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177365.500947:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177365.500949:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e680.
00000100:00000001:2.0:1786177365.500951:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177365.500953:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.500954:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177365.500957:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ed80 x1872934856370048/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500962:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177365.500963:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177365.500964:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177365.500966:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177365.500967:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177365.500968:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177365.500970:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177365.500971:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:2.0:1786177365.500973:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177365.500974:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177365.500975:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ed80.
00000100:00000001:2.0:1786177365.500978:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177365.500979:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.500980:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177365.500983:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c380 x1872934856370176/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.500988:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177365.500989:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177365.500991:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab0000.
02000000:00000001:2.0:1786177365.500994:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177365.500995:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177365.500996:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177365.500998:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177365.500999:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:2.0:1786177365.501001:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177365.501002:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177365.501003:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c380.
00000100:00000001:2.0:1786177365.501005:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177365.501006:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.501007:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177365.501010:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d880 x1872934856370304/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177365.501015:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177365.501017:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177365.501018:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab1800.
02000000:00000001:2.0:1786177365.501020:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177365.501021:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177365.501022:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177365.501024:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177365.501025:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:2.0:1786177365.501027:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177365.501028:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177365.501029:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d880.
00000100:00000001:2.0:1786177365.501031:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177365.501032:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.501034:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177365.501039:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177365.501041:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177365.501043:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177365.501044:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177366.072626:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177366.072633:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177366.072639:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177366.072640:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177366.072644:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177366.072645:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177366.073347:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177366.073351:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177366.073354:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177366.520776:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177366.520787:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177366.520789:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177366.520796:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177366.520801:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177366.520805:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177366.520809:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177366.520814:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177366.520817:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177367.096611:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177367.096618:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177367.096623:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177367.096624:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177367.096627:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177367.096629:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177367.096742:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177367.096749:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177367.096753:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177367.544671:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177367.544681:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177367.544684:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177367.544691:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177367.544696:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177367.544701:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177367.544705:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177367.544708:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177367.544712:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177368.120626:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177368.120633:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177368.120639:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177368.120640:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177368.120643:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177368.120644:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177368.120887:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177368.120893:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177368.120897:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177368.568695:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177368.568702:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177368.568704:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177368.568710:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177368.568713:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177368.568717:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177368.568719:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177368.568722:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177368.568725:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177369.148195:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177369.148203:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177369.148209:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177369.148210:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177369.148213:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177369.148214:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177369.148626:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177369.148631:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177369.148635:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177369.336759:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302265038, at 4302265088
00000100:00000010:1.0:1786177369.336796:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25960.
00000100:00000040:0.0:1786177369.337470:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302265038, at 4302265088
00000100:00000010:0.0:1786177369.337481:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25ea0.
00000100:00000001:0.0:1786177369.592714:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177369.592728:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177369.592732:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177369.592743:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177369.592751:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177369.592758:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177369.592872:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177369.592877:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177369.592882:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177370.168680:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177370.168692:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177370.168700:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177370.168701:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177370.168705:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177370.168707:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177370.168790:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177370.168795:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177370.168799:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1786177370.616662:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177370.616677:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177370.616680:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177370.616685:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177370.616690:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177370.616700:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7593
00000400:00000200:3.0:1786177370.616712:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177370.616719:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177370.616724:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177370.616728:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.616732:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177370.616734:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.616737:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177370.616742:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177370.616744:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177370.616747:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177370.616751:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177370.616753:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177370.616756:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177370.616759:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.616762:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177370.616763:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.616765:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177370.616769:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177370.616772:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177370.616775:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177370.616787:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177370.616791:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177370.616794:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.616796:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177370.616799:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177370.616811:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69fb80.
00000020:00000040:3.0:1786177370.616815:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177370.616820:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177370.616822:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.616825:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177370.616829:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177370.616834:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177370.616843:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786177370.616851:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177370.616863:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177370.616875:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177370.616880:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177370.616885:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000014f55bbb] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177370.616918:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177370.616922:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177370.616926:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177370.616928:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177370.616930:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177370.616933:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177370.616936:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7593
00000400:00000200:3.0:1786177370.616940:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177370.616944:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177370.616946:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177370.617029:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617031:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177370.617034:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617036:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177370.617039:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177370.617040:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177370.617042:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177370.617131:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786177370.617135:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786177370.617137:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177370.617141:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617144:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177370.617145:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617148:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177370.617151:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786177370.617154:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617156:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177370.617158:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177370.617165:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177370.617167:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.617170:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177370.617174:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177370.617178:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.617180:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177370.617184:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ca80.
00000020:00000040:3.0:1786177370.617187:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177370.617190:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177370.617191:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.617193:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177370.617195:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177370.617197:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177370.617201:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786177370.617204:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177370.617210:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177370.617218:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177370.617220:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177370.617224:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ffb0865] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177370.617233:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177370.617236:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177370.617239:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177370.617241:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177370.617243:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177370.617245:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177370.617248:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7593
00000400:00000200:3.0:1786177370.617253:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177370.617256:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177370.617259:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177370.617262:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617264:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177370.617266:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617269:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177370.617271:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177370.617272:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177370.617274:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177370.617276:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786177370.617279:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177370.617281:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177370.617284:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617286:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177370.617288:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617290:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177370.617293:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786177370.617295:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617298:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177370.617300:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177370.617302:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177370.617304:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.617307:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177370.617309:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177370.617312:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.617313:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177370.617317:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f800.
00000020:00000040:3.0:1786177370.617319:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177370.617322:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177370.617324:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.617325:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177370.617327:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177370.617328:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177370.617334:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:3.0:1786177370.617337:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177370.617342:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177370.617350:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177370.617352:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177370.617356:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4d339d9] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177370.617364:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177370.617367:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177370.617371:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177370.617372:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177370.617374:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177370.617377:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177370.617379:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7593
00000400:00000200:3.0:1786177370.617384:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177370.617387:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177370.617389:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177370.617392:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617394:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177370.617396:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617399:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177370.617401:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177370.617402:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177370.617404:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177370.617406:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177370.617408:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177370.617411:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177370.617414:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617416:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177370.617418:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617421:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177370.617423:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177370.617426:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617428:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177370.617430:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177370.617433:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177370.617435:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.617437:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177370.617440:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177370.617443:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.617444:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177370.617448:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f480.
00000020:00000040:3.0:1786177370.617450:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177370.617452:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177370.617454:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.617455:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177370.617456:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177370.617458:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177370.617462:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:3.0:1786177370.617465:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177370.617471:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177370.617479:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177370.617481:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177370.617485:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092aa1aa7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177370.617493:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177370.617496:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177370.617500:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177370.617502:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177370.617503:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177370.617506:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177370.617508:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7593
00000400:00000200:3.0:1786177370.617550:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177370.617553:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177370.617556:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177370.617558:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617561:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177370.617563:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617565:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177370.617567:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177370.617569:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177370.617571:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177370.617573:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177370.617576:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177370.617578:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177370.617581:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617583:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177370.617585:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617587:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177370.617589:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786177370.617592:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177370.617594:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177370.617596:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177370.617599:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177370.617602:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.617604:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177370.617607:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177370.617609:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.617611:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177370.617614:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69f100.
00000020:00000040:3.0:1786177370.617616:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177370.617619:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177370.617620:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177370.617622:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177370.617623:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177370.617625:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177370.617629:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:3.0:1786177370.617633:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177370.617637:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177370.617645:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177370.617648:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177370.617651:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1ef9f45] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177370.617660:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177370.617665:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7604)
00000100:00000001:2.0:1786177370.617853:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177370.617858:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177370.617861:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177370.617868:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.617876:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177370.617878:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177370.617881:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177370.617884:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177370.617886:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177370.617889:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856370432:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177370.617893:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1786177370.617894:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
02000000:00000001:2.0:1786177370.617898:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1786177370.617900:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:2.0:1786177370.617902:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177370.617904:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177370.617905:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:0.0:1786177370.617910:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
02000000:00000010:2.0:1786177370.617912:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:0.0:1786177370.617915:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1786177370.617916:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177370.617921:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1786177370.617925:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177370.617927:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000400:00000010:2.0:1786177370.617930:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000100:00000001:0.0:1786177370.617931:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177370.617936:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1786177370.617938:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177370.617947:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856370432, portal 25
00000100:00000001:2.0:1786177370.617949:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177370.617950:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764307840 : -114547945243776 : ffff97d1ba69fb80)
00000100:00000040:2.0:1786177370.617954:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.617960:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177370.617962:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177370.617964:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177370.617967:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856370432, offset 0
00000400:00000200:2.0:1786177370.617972:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177370.617980:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177370.617984:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177370.617989:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177370.617993:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177370.617997:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177370.617999:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177370.618002:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255065:255065:256:4294967295] 192.168.202.110@tcp LPNI seq info [255054:255054:8:4294967295]
00000400:00000200:2.0:1786177370.618006:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177370.618014:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177370.618018:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177370.618023:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177370.618029:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177370.618033:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1786177370.618037:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177370.618040:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1786177370.618043:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177370.618046:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177370.618054:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177370.618058:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177370.618065:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177370.618068:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177370.618073:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177370.618077:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618098:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177370.618102:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618110:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177370.618112:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177370.618121:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177370.618125:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177370.618128:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302270158
00000100:00000010:2.0:1786177370.618135:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25ae0.
00000100:00000001:2.0:1786177370.618143:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177370.618150:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302270158
00000100:00000010:2.0:1786177370.618154:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25ea0.
00000100:00000001:2.0:1786177370.618157:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786177370.618161:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177370.618164:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177370.618167:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177370.618172:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177370/real 1786177370]  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177370.618183:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177370.618185:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177370.618187:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177370.618189:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177370.618190:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177370.618192:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177370.618195:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177370.618201:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618206:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177370.618207:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618208:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177370.618210:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177370.618213:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618217:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177370.618220:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618229:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177370.618231:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618232:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177370.618233:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618235:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177370.618236:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177370.618238:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177370.618241:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177370.618242:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177370.618245:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177370.618249:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856370432:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177370.618252:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177370.618254:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618259:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177370.618260:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177370.618261:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177370.618262:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177370.618263:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177370.618265:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177370.618267:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177370.618269:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177370.618270:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177370.618271:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177370.618274:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177370.618275:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177370.618277:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177370.618279:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177370.618281:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177370.618283:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856370560, portal 10
00000100:00000001:2.0:1786177370.618284:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177370.618285:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764295296 : -114547945256320 : ffff97d1ba69ca80)
00000100:00000040:2.0:1786177370.618290:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618294:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177370.618296:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177370.618297:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177370.618298:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856370560, offset 0
00000400:00000200:2.0:1786177370.618301:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177370.618303:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177370.618305:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177370.618307:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177370.618309:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177370.618310:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177370.618312:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177370.618314:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255066:255066:256:4294967295] 192.168.202.110@tcp LPNI seq info [255055:255055:8:4294967295]
00000400:00000200:2.0:1786177370.618316:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177370.618321:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177370.618324:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177370.618326:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177370.618329:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177370.618330:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1786177370.618333:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177370.618335:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1786177370.618338:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177370.618339:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177370.618342:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177370.618344:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177370.618346:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177370.618347:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177370.618349:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177370.618351:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618356:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177370.618358:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618362:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177370.618363:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177370.618365:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177370.618368:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:2.0:1786177370.618370:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177370.618371:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302266378
00000100:00000001:2.0:1786177370.618373:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302270158
00000100:00000001:2.0:1786177370.618375:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302266378
00000100:00000001:2.0:1786177370.618376:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302270158
00000100:00000001:2.0:1786177370.618377:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177370.618379:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177370.618380:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177370.618382:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177370/real 1786177370]  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177370.618386:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177370.618387:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177370.618388:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177370.618389:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177370.618390:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177370.618391:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177370.618393:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177370.618398:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618406:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177370.618407:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618408:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177370.618409:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177370.618410:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618414:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177370.618416:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618419:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177370.618420:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618421:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177370.618422:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618423:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177370.618424:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177370.618426:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177370.618427:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177370.618428:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177370.618430:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177370.618435:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370560:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177370.618437:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177370.618438:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618442:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177370.618443:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177370.618445:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177370.618446:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177370.618447:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177370.618449:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177370.618451:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177370.618452:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177370.618453:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177370.618454:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177370.618456:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177370.618457:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177370.618459:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177370.618460:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177370.618462:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177370.618464:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856370688, portal 10
00000100:00000001:2.0:1786177370.618465:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177370.618466:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306944 : -114547945244672 : ffff97d1ba69f800)
00000100:00000040:2.0:1786177370.618470:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618474:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177370.618476:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177370.618477:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177370.618479:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856370688, offset 0
00000400:00000200:2.0:1786177370.618481:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177370.618482:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177370.618484:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177370.618486:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177370.618487:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177370.618489:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177370.618490:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177370.618492:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255067:255067:256:4294967295] 192.168.202.110@tcp LPNI seq info [255056:255056:8:4294967295]
00000400:00000200:2.0:1786177370.618495:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177370.618499:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177370.618503:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177370.618505:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177370.618507:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177370.618509:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1786177370.618536:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177370.618538:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1786177370.618540:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177370.618541:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177370.618543:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177370.618545:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177370.618547:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177370.618548:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177370.618550:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177370.618552:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618557:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177370.618559:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618563:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177370.618565:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177370.618566:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177370.618568:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:2.0:1786177370.618570:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177370.618571:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302266504
00000100:00000001:2.0:1786177370.618573:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302270158
00000100:00000001:2.0:1786177370.618575:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302266504
00000100:00000001:2.0:1786177370.618576:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302270158
00000100:00000001:2.0:1786177370.618579:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177370.618581:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177370.618582:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177370.618586:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177370/real 1786177370]  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177370.618593:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177370.618594:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177370.618596:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177370.618598:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177370.618600:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177370.618602:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177370.618604:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177370.618613:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618619:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177370.618621:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618623:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177370.618624:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177370.618628:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618634:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177370.618638:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618642:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177370.618644:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618645:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177370.618646:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618647:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177370.618648:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177370.618649:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177370.618651:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177370.618652:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177370.618655:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177370.618660:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370688:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177370.618662:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177370.618664:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618668:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177370.618668:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177370.618670:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177370.618671:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177370.618673:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177370.618674:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177370.618676:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177370.618678:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177370.618679:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177370.618680:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177370.618682:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177370.618684:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177370.618686:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177370.618688:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177370.618691:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177370.618692:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856370816, portal 4
00000100:00000001:2.0:1786177370.618694:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177370.618695:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764306048 : -114547945245568 : ffff97d1ba69f480)
00000100:00000040:2.0:1786177370.618700:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618704:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177370.618705:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177370.618706:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177370.618708:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856370816, offset 0
00000400:00000200:2.0:1786177370.618711:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177370.618714:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177370.618716:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177370.618718:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177370.618720:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177370.618722:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177370.618723:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177370.618725:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255068:255068:256:4294967295] 192.168.202.110@tcp LPNI seq info [255057:255057:8:4294967295]
00000400:00000200:2.0:1786177370.618728:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177370.618731:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177370.618734:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177370.618736:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177370.618738:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177370.618740:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1786177370.618742:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177370.618744:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1786177370.618746:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177370.618747:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177370.618750:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177370.618752:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177370.618754:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177370.618755:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177370.618756:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177370.618758:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618763:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177370.618765:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618769:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177370.618770:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177370.618772:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177370.618774:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:2.0:1786177370.618776:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177370.618778:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302266379
00000100:00000001:2.0:1786177370.618780:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302270158
00000100:00000001:2.0:1786177370.618781:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302266379
00000100:00000001:2.0:1786177370.618783:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302270158
00000100:00000001:2.0:1786177370.618784:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177370.618785:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177370.618786:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177370.618788:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177370/real 1786177370]  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177370.618791:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177370.618793:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177370.618794:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177370.618795:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177370.618796:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177370.618797:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177370.618799:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177370.618803:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618807:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177370.618807:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618808:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177370.618809:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177370.618811:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618814:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177370.618816:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618820:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177370.618822:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618823:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177370.618824:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618825:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177370.618826:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177370.618827:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177370.618829:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177370.618830:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177370.618832:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177370.618836:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370816:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177370.618837:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177370.618839:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618842:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177370.618843:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177370.618844:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177370.618845:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177370.618846:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177370.618848:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177370.618849:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177370.618850:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177370.618851:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177370.618852:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177370.618854:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab0800.
02000000:00000001:2.0:1786177370.618855:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177370.618857:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177370.618859:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177370.618860:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177370.618862:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856370944, portal 4
00000100:00000001:2.0:1786177370.618864:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177370.618865:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764305152 : -114547945246464 : ffff97d1ba69f100)
00000100:00000040:2.0:1786177370.618868:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618872:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177370.618873:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177370.618874:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177370.618875:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856370944, offset 0
00000400:00000200:2.0:1786177370.618877:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177370.618879:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177370.618881:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177370.618883:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177370.618884:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177370.618886:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177370.618887:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177370.618890:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255069:255069:256:4294967295] 192.168.202.110@tcp LPNI seq info [255058:255058:8:4294967295]
00000400:00000200:2.0:1786177370.618892:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177370.618895:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177370.618897:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177370.618899:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177370.618901:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177370.618902:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1786177370.618905:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177370.618906:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1786177370.618908:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177370.618910:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177370.618911:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177370.618913:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177370.618915:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177370.618916:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177370.618917:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177370.618919:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618927:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177370.618929:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618933:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177370.618934:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177370.618935:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177370.618937:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:2.0:1786177370.618938:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177370.618939:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302266387
00000100:00000001:2.0:1786177370.618940:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302270158
00000100:00000001:2.0:1786177370.618942:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302266387
00000100:00000001:2.0:1786177370.618943:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302270158
00000100:00000001:2.0:1786177370.618945:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177370.618946:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177370.618947:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177370.618949:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177370/real 1786177370]  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177370.618953:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177370.618953:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177370.618954:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177370.618955:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177370.618956:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177370.618957:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177370.618958:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177370.618963:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618967:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177370.618968:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618969:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177370.618970:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177370.618972:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618976:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177370.618978:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.618981:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177370.618982:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618983:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177370.618984:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618984:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177370.618985:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177370.618987:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177370.618988:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177370.618989:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177370.618991:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177370.618995:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856370944:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177370.618997:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.618999:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177370.619001:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69fb80 x1872934856370432/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.619005:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177370.619008:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177370.619010:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab0000.
02000000:00000001:2.0:1786177370.619013:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177370.619015:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177370.619016:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1786177370.619018:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177370.619020:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:2.0:1786177370.619023:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177370.619025:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177370.619027:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69fb80.
00000100:00000001:2.0:1786177370.619029:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177370.619030:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.619031:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177370.619033:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ca80 x1872934856370560/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.619036:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177370.619037:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177370.619038:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3000.
02000000:00000001:2.0:1786177370.619039:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177370.619040:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177370.619040:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177370.619042:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177370.619042:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:2.0:1786177370.619043:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177370.619044:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177370.619044:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ca80.
00000100:00000001:2.0:1786177370.619046:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177370.619046:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.619047:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177370.619049:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f800 x1872934856370688/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.619052:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177370.619052:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177370.619053:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177370.619055:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177370.619056:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177370.619056:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177370.619058:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177370.619059:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:2.0:1786177370.619061:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177370.619063:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177370.619064:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f800.
00000100:00000001:2.0:1786177370.619066:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177370.619067:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.619069:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177370.619072:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f480 x1872934856370816/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.619078:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177370.619079:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177370.619081:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177370.619084:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177370.619085:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177370.619211:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177370.619214:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177370.619216:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:2.0:1786177370.619218:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177370.619219:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177370.619220:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f480.
00000100:00000001:2.0:1786177370.619221:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177370.619222:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.619224:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177370.619227:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69f100 x1872934856370944/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177370.619233:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177370.619234:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177370.619236:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab0800.
02000000:00000001:2.0:1786177370.619238:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177370.619239:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177370.619240:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177370.619242:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177370.619243:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:2.0:1786177370.619246:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177370.619247:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177370.619248:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69f100.
00000100:00000001:2.0:1786177370.619250:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177370.619251:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.619254:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177370.619265:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177370.619267:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177370.619268:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177370.619270:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177371.193304:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177371.193315:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177371.193321:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177371.193323:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177371.193326:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177371.193328:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177371.193542:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177371.193547:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177371.193551:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177371.640682:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177371.640693:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177371.640696:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177371.640703:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177371.640708:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177371.640712:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177371.640715:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177371.640718:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177371.640721:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177372.216686:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177372.216694:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177372.216696:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177372.216700:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177372.216706:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177372.216708:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177372.216710:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177372.216713:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177372.216717:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177372.216718:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177372.216720:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177372.216721:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177372.216723:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177372.216724:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177372.216726:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177372.216728:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177372.216731:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177372.216733:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177372.216833:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177372.216840:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177372.216845:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177372.664981:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177372.664994:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177372.664996:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177372.665003:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177372.665009:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177372.665012:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177372.665015:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177372.665018:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177372.665021:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177373.240681:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177373.240689:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177373.240697:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177373.240698:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177373.240701:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177373.240702:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177373.241006:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177373.241013:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177373.241018:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177373.688669:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177373.688680:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177373.688683:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177373.688691:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177373.688696:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177373.688700:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177373.688704:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177373.688707:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177373.688710:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177374.268811:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177374.268821:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177374.268827:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177374.268829:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177374.268833:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177374.268834:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177374.268905:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177374.268910:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177374.268913:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177374.456651:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302270158, at 4302270208
00000100:00000040:1.0:1786177374.456658:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302270158, at 4302270208
00000100:00000010:0.0:1786177374.456672:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25ae0.
00000100:00000010:1.0:1786177374.456676:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25ea0.
00000800:00000200:2.0:1786177374.585767:0:2383:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d188a63d80] -> 12345-192.168.202.10@tcp (6)
00000800:00000200:2.0:1786177374.585779:0:2383:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.10@tcp ip 192.168.202.10:988
00000800:00000200:2.0:1786177374.585785:0:2383:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff97d18376f400 type 192, nob 24 niov 1 nkiov 0
00000800:00000200:0.0:1786177374.586289:0:2384:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000001:0.0:1786177374.586297:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786177374.586311:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000800:00000001:0.0:1786177374.586318:0:2384:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered
00000800:00000001:0.0:1786177374.586321:0:2384:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177374.712722:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177374.712735:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177374.712738:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177374.712745:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177374.712752:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177374.712757:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177374.712761:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177374.712765:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177374.712769:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177375.292334:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177375.292343:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177375.292361:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177375.292363:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177375.292367:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177375.292371:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177375.292374:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177375.292377:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177375.292381:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177375.292386:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177375.292388:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177375.292390:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177375.293256:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177375.293261:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177375.293265:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1786177375.736871:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177375.736884:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1786177375.736887:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177375.736892:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177375.736897:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177375.736906:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7599
00000400:00000200:0.0:1786177375.736916:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177375.736925:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177375.736929:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177375.736932:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.736936:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177375.736938:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.736941:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177375.736945:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177375.736948:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:0.0:1786177375.736951:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177375.736955:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:0.0:1786177375.736957:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:0.0:1786177375.736960:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:0.0:1786177375.736964:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.736966:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177375.736967:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.736970:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177375.736973:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:0.0:1786177375.736977:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177375.736980:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177375.736983:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177375.736987:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177375.736991:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.736994:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:0.0:1786177375.736996:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177375.737008:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418000.
00000020:00000040:0.0:1786177375.737012:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:0.0:1786177375.737016:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177375.737018:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737022:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177375.737025:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177375.737031:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177375.737073:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a7800.
00000100:00000001:0.0:1786177375.737082:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177375.737093:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177375.737104:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177375.737109:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177375.737114:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002e10e4f5] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177375.737147:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177375.737151:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177375.737154:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1786177375.737155:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177375.737157:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177375.737159:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177375.737162:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7599
00000400:00000200:0.0:1786177375.737166:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177375.737169:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177375.737171:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177375.737175:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737177:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177375.737178:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737180:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177375.737183:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177375.737184:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:0.0:1786177375.737186:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177375.737187:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:0.0:1786177375.737189:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:0.0:1786177375.737191:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:0.0:1786177375.737194:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737196:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177375.737198:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737200:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177375.737202:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:0.0:1786177375.737204:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737206:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177375.737207:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177375.737213:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177375.737215:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737217:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177375.737222:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177375.737225:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737227:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177375.737231:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41b800.
00000020:00000040:0.0:1786177375.737233:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177375.737235:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177375.737236:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737238:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177375.737239:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177375.737241:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177375.737245:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a5800.
00000100:00000001:0.0:1786177375.737248:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177375.737253:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177375.737260:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177375.737262:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177375.737266:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000192ee9a0] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177375.737273:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177375.737275:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177375.737278:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1786177375.737280:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177375.737281:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177375.737284:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177375.737286:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7599
00000400:00000200:0.0:1786177375.737290:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177375.737292:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1786177375.737294:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177375.737297:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737299:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177375.737300:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737302:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177375.737305:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:0.0:1786177375.737306:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:0.0:1786177375.737308:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177375.737310:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:0.0:1786177375.737312:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:0.0:1786177375.737314:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:0.0:1786177375.737317:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737318:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177375.737321:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737322:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177375.737325:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:0.0:1786177375.737327:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737330:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177375.737331:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177375.737333:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177375.737335:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737337:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177375.737339:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1786177375.737341:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737342:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177375.737345:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba419c00.
00000020:00000040:0.0:1786177375.737348:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:0.0:1786177375.737360:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1786177375.737361:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737363:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177375.737365:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177375.737366:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177375.737372:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6000.
00000100:00000001:0.0:1786177375.737376:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177375.737380:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1786177375.737387:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1786177375.737389:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.737391:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1786177375.737392:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cf890c1c] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.737396:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177375.737399:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.737399:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1786177375.737402:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177375.737405:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000040:2.0:1786177375.737406:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177375.737407:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177375.737409:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1786177375.737412:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177375.737414:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7599
00000100:00000001:2.0:1786177375.737414:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177375.737417:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1786177375.737418:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177375.737420:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
02000000:00000001:2.0:1786177375.737421:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1786177375.737422:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
02000000:00000001:2.0:1786177375.737423:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1786177375.737425:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:2.0:1786177375.737426:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737427:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177375.737429:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00100000:2.0:1786177375.737429:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856371072:192.168.202.110@tcp:250:kworker.0
00000100:00000001:0.0:1786177375.737432:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177375.737434:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000100:00000001:2.0:1786177375.737434:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:0.0:1786177375.737435:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:0.0:1786177375.737436:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177375.737438:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
02000000:00000001:2.0:1786177375.737439:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:0.0:1786177375.737440:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:0.0:1786177375.737442:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
02000000:00000001:2.0:1786177375.737442:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1786177375.737445:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
02000000:00000001:2.0:1786177375.737445:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1786177375.737447:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177375.737449:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
02000000:00000010:2.0:1786177375.737450:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:0.0:1786177375.737451:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:0.0:1786177375.737453:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
02000000:00000001:2.0:1786177375.737453:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177375.737455:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737458:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177375.737458:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
02000000:00000001:0.0:1786177375.737459:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177375.737462:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:2.0:1786177375.737462:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
02000000:00000001:0.0:1786177375.737464:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737465:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:0.0:1786177375.737468:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177375.737470:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1786177375.737470:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000001:0.0:1786177375.737472:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177375.737475:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba41bb80.
00000020:00000040:0.0:1786177375.737477:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177375.737479:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:2.0:1786177375.737479:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856371072, portal 25
02000000:00000001:0.0:1786177375.737480:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737482:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1786177375.737482:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1786177375.737483:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1786177375.737484:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761654784 : -114547947896832 : ffff97d1ba418000)
02000000:00000001:0.0:1786177375.737485:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1786177375.737489:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a6800.
00000100:00000040:2.0:1786177375.737489:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177375.737492:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177375.737496:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177375.737497:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177375.737500:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737502:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1786177375.737503:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000001:0.0:1786177375.737504:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1786177375.737507:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856371072, offset 0
00000100:00000040:0.0:1786177375.737508:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9983a05] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177375.737533:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1786177375.737536:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1786177375.737539:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1786177375.737540:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1786177375.737542:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1786177375.737543:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737544:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1786177375.737546:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7599
00000400:00000200:0.0:1786177375.737549:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:0.0:1786177375.737552:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1786177375.737552:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00080000:0.0:1786177375.737554:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1786177375.737556:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737558:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1786177375.737558:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000100:00000040:0.0:1786177375.737560:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737562:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:0.0:1786177375.737563:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000400:00000200:2.0:1786177375.737564:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:0.0:1786177375.737565:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:0.0:1786177375.737566:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:0.0:1786177375.737569:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000400:00000200:2.0:1786177375.737569:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000020:00000001:0.0:1786177375.737571:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:0.0:1786177375.737573:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000400:00000200:2.0:1786177375.737574:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1786177375.737576:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000400:00000200:2.0:1786177375.737577:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737578:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1786177375.737579:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:0.0:1786177375.737581:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000400:00000200:2.0:1786177375.737581:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255070:255070:256:4294967295] 192.168.202.110@tcp LPNI seq info [255059:255059:8:4294967295]
00000020:00000040:0.0:1786177375.737583:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:0.0:1786177375.737585:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000400:00000200:2.0:1786177375.737585:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:0.0:1786177375.737587:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1786177375.737588:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1786177375.737591:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1786177375.737592:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737594:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000400:00000200:2.0:1786177375.737596:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000008:00100000:0.0:1786177375.737597:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1786177375.737599:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737600:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1786177375.737603:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba418700.
00000800:00000200:2.0:1786177375.737603:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000020:00000040:0.0:1786177375.737605:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:0.0:1786177375.737607:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000010:2.0:1786177375.737608:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
02000000:00000001:0.0:1786177375.737609:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1786177375.737610:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1786177375.737611:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1786177375.737613:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786177375.737615:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
02000000:00000010:0.0:1786177375.737617:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1885a4000.
00000100:00000001:0.0:1786177375.737620:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1786177375.737620:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00100000:0.0:1786177375.737624:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1786177375.737626:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000100:00000001:0.0:1786177375.737630:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000800:00000200:2.1:1786177375.737630:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1786177375.737632:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:2.0:1786177375.737634:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000100:00000040:0.0:1786177375.737636:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bf66f89d] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:2.0:1786177375.737637:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000100:00000001:0.0:1786177375.737642:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177375.737646:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1786177375.737647:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7609)
00000400:00000200:2.0:1786177375.737651:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177375.737656:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177375.737659:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177375.737664:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177375.737667:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.737677:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177375.737680:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.737687:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.737689:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177375.737697:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177375.737701:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177375.737705:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302275278
00000100:00000010:2.0:1786177375.737711:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f25960.
00000100:00000001:2.0:1786177375.737719:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:2.0:1786177375.737725:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302275278
00000100:00000010:2.0:1786177375.737728:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1b9f254e0.
00000100:00000001:2.0:1786177375.737731:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:2.0:1786177375.737735:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177375.737737:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.737740:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177375.737744:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177375/real 1786177375]  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177375.737744:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177375.737748:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177375.737750:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177375.737754:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177375.737756:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177375.737759:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177375.737761:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177375.737762:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177375.737766:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1786177375.737819:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177375.737822:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177375.737824:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177375.737828:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177375.737830:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177375.737833:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177375.737837:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177375.737845:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.737851:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177375.737854:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.737857:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177375.737859:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177375.737863:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.737870:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177375.737876:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.737883:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177375.737886:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.737888:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177375.737889:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.737892:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177375.737894:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177375.737897:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177375.737900:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177375.737903:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177375.737908:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177375.737917:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856371072:192.168.202.110@tcp:250:kworker.0
00000100:00000001:2.0:1786177375.737922:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177375.737925:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.737932:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177375.737933:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177375.737935:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177375.737937:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177375.737939:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177375.737942:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856371200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177375.737945:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177375.737947:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177375.737949:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177375.737951:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177375.737955:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177375.737957:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177375.737960:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177375.737963:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177375.737967:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177375.737970:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856371200, portal 10
00000100:00000001:2.0:1786177375.737971:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177375.737973:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761669120 : -114547947882496 : ffff97d1ba41b800)
00000100:00000040:2.0:1786177375.737979:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.737985:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177375.737987:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177375.737989:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177375.737992:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856371200, offset 0
00000400:00000200:2.0:1786177375.737996:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177375.737999:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177375.738003:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177375.738006:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177375.738008:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177375.738011:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177375.738013:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177375.738016:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255071:255071:256:4294967295] 192.168.202.110@tcp LPNI seq info [255060:255060:8:4294967295]
00000400:00000200:2.0:1786177375.738020:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177375.738026:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177375.738031:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177375.738035:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177375.738051:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177375.738054:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1786177375.738059:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177375.738063:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1786177375.738066:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177375.738068:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177375.738071:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177375.738074:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177375.738077:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177375.738079:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177375.738081:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177375.738084:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738091:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177375.738094:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738099:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.738100:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177375.738103:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177375.738107:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:2.0:1786177375.738110:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177375.738112:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302271498
00000100:00000001:2.0:1786177375.738115:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302275278
00000100:00000001:2.0:1786177375.738116:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302271498
00000100:00000001:2.0:1786177375.738118:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302275278
00000100:00000001:2.0:1786177375.738120:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177375.738122:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.738124:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177375.738128:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177375/real 1786177375]  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177375.738136:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177375.738137:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177375.738139:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177375.738141:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177375.738143:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177375.738145:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177375.738148:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177375.738157:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738163:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177375.738165:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.738166:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177375.738167:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177375.738171:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738177:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177375.738181:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738187:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177375.738190:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.738191:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177375.738192:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.738194:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177375.738196:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177375.738199:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177375.738202:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177375.738204:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177375.738207:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177375.738215:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856371200:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177375.738219:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.738221:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177375.738224:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418000 x1872934856371072/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738231:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177375.738235:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177375.738238:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab0800.
02000000:00000001:2.0:1786177375.738243:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177375.738245:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177375.738247:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:2.0:1786177375.738250:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177375.738253:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a7800.
02000000:00000001:2.0:1786177375.738257:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177375.738259:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177375.738261:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418000.
00000100:00000001:2.0:1786177375.738265:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177375.738266:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.738268:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177375.738271:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41b800 x1872934856371200/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738276:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177375.738278:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177375.738279:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177375.738281:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177375.738282:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177375.738283:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177375.738285:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177375.738286:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a5800.
02000000:00000001:2.0:1786177375.738288:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177375.738289:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177375.738290:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41b800.
00000100:00000001:2.0:1786177375.738293:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177375.738294:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.738296:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.738304:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177375.738306:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.738308:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177375.738310:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1786177375.738311:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177375.738314:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738319:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177375.738320:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177375.738322:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177375.738324:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177375.738326:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177375.738328:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856371328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177375.738331:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177375.738333:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177375.738334:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177375.738336:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177375.738339:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177375.738341:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177375.738344:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177375.738347:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177375.738350:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177375.738353:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856371328, portal 10
00000100:00000001:2.0:1786177375.738355:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177375.738356:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761661952 : -114547947889664 : ffff97d1ba419c00)
00000100:00000040:2.0:1786177375.738363:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738370:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177375.738372:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177375.738374:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177375.738377:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856371328, offset 0
00000400:00000200:2.0:1786177375.738381:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177375.738385:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177375.738388:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177375.738391:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177375.738394:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177375.738396:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177375.738399:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177375.738401:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255072:255072:256:4294967295] 192.168.202.110@tcp LPNI seq info [255061:255061:8:4294967295]
00000400:00000200:2.0:1786177375.738405:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177375.738411:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177375.738416:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177375.738420:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177375.738424:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177375.738426:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1786177375.738430:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177375.738433:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1786177375.738437:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177375.738438:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177375.738442:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177375.738445:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177375.738448:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177375.738449:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177375.738452:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177375.738456:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738464:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177375.738467:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738473:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.738475:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177375.738478:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177375.738481:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:2.0:1786177375.738484:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177375.738486:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302271623
00000100:00000001:2.0:1786177375.738488:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302275278
00000100:00000001:2.0:1786177375.738491:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302271623
00000100:00000001:2.0:1786177375.738493:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302275278
00000100:00000001:2.0:1786177375.738495:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177375.738497:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.738499:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177375.738503:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177375/real 1786177375]  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177375.738540:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177375.738541:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177375.738543:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177375.738547:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177375.738548:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177375.738551:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177375.738555:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177375.738563:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738569:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177375.738570:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.738572:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177375.738573:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177375.738578:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738584:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177375.738587:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738594:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177375.738596:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.738597:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177375.738599:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.738601:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177375.738603:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177375.738605:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177375.738608:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177375.738610:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177375.738614:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177375.738622:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba419c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856371328:192.168.202.110@tcp:38:kworker.0
00000100:00000001:2.0:1786177375.738626:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177375.738630:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738636:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177375.738637:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177375.738640:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177375.738641:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177375.738643:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177375.738646:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856371456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177375.738649:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177375.738651:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177375.738652:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177375.738653:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177375.738657:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab0800.
02000000:00000001:2.0:1786177375.738659:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177375.738662:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177375.738664:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177375.738667:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177375.738670:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856371456, portal 4
00000100:00000001:2.0:1786177375.738672:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177375.738673:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761670016 : -114547947881600 : ffff97d1ba41bb80)
00000100:00000040:2.0:1786177375.738679:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.738685:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177375.738687:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177375.738689:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177375.738691:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856371456, offset 0
00000400:00000200:2.0:1786177375.738695:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177375.738699:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177375.738703:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177375.738705:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177375.738708:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177375.738710:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177375.738713:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177375.738715:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255073:255073:256:4294967295] 192.168.202.110@tcp LPNI seq info [255062:255062:8:4294967295]
00000400:00000200:2.0:1786177375.738719:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177375.738725:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177375.738729:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177375.738732:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177375.738736:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177375.738738:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1786177375.738742:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177375.738745:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1786177375.738748:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177375.738750:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177375.739399:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177375.739407:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177375.739411:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177375.739413:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177375.739417:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177375.739421:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.739572:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177375.739577:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.739583:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.739585:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177375.739590:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177375.739594:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:2.0:1786177375.739597:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177375.739599:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302271500
00000100:00000001:2.0:1786177375.739602:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302275278
00000100:00000001:2.0:1786177375.739604:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302271500
00000100:00000001:2.0:1786177375.739607:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302275278
00000100:00000001:2.0:1786177375.739609:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177375.739611:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.739613:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177375.739617:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177375/real 1786177375]  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177375.739625:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177375.739626:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177375.739628:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177375.739631:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177375.739633:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177375.739635:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177375.739638:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177375.739646:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.739653:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177375.739654:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.739656:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177375.739657:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177375.739661:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.739667:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177375.739671:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.739677:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177375.739679:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.739681:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177375.739682:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.739685:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177375.739686:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177375.739690:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177375.739694:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177375.739696:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177375.739700:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177375.739707:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba41bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856371456:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177375.739711:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1786177375.739715:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.739722:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1786177375.739723:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177375.739726:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1786177375.739728:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177375.739730:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177375.739733:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856371584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177375.739736:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1786177375.739739:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1786177375.739740:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177375.739742:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1786177375.739747:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177375.739749:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1786177375.739753:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:2.0:1786177375.739826:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d1a99a7b00.
00000400:00000010:2.0:1786177375.739830:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2c38.
00000100:00000200:2.0:1786177375.739834:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856371584, portal 4
00000100:00000001:2.0:1786177375.739837:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1786177375.739838:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525761656576 : -114547947895040 : ffff97d1ba418700)
00000100:00000040:2.0:1786177375.739846:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.739853:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1786177375.739855:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:2.0:1786177375.739857:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d1831a2220.
00000100:00000200:2.0:1786177375.739861:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856371584, offset 0
00000400:00000200:2.0:1786177375.739865:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:2.0:1786177375.739870:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177375.739874:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:2.0:1786177375.739877:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1786177375.739881:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:2.0:1786177375.739884:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1786177375.739887:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:2.0:1786177375.739889:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255074:255074:256:4294967295] 192.168.202.110@tcp LPNI seq info [255063:255063:8:4294967295]
00000400:00000200:2.0:1786177375.739894:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177375.739901:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1786177375.739906:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177375.739910:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d1833efb00.
00000800:00000200:2.0:1786177375.739915:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.0:1786177375.739918:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1786177375.739923:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:2.1:1786177375.739925:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1786177375.739929:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:2.0:1786177375.739931:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d1833efb00.
00000400:00000200:2.0:1786177375.739935:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1786177375.739938:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1786177375.739941:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2220
00000400:00000010:2.0:1786177375.739942:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2220.
00000100:00000001:2.0:1786177375.739944:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1786177375.739948:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.739956:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177375.739958:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.739964:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.739965:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1786177375.739968:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177375.739971:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:2.0:1786177375.739975:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:2.0:1786177375.739977:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302271508
00000100:00000001:2.0:1786177375.739979:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302275278
00000100:00000001:2.0:1786177375.739981:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302271508
00000100:00000001:2.0:1786177375.739984:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302275278
00000100:00000001:2.0:1786177375.739986:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1786177375.739988:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.739990:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1786177375.739993:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177375/real 1786177375]  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1786177375.740000:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1786177375.740001:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1786177375.740003:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d1a99a7b00.
00000400:00000200:2.0:1786177375.740005:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d1831a2c38
00000400:00000010:2.0:1786177375.740007:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d1831a2c38.
00000100:00000001:2.0:1786177375.740009:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1786177375.740012:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1786177375.740019:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.740025:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1786177375.740027:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.740029:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177375.740030:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1786177375.740033:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.740050:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1786177375.740054:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.740061:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1786177375.740063:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.740065:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1786177375.740066:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.740068:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1786177375.740069:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1786177375.740072:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1786177375.740076:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:2.0:1786177375.740078:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1786177375.740081:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1786177375.740089:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba418700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856371584:192.168.202.110@tcp:8:kworker.0
00000100:00000001:2.0:1786177375.740093:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.740096:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177375.740099:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba419c00 x1872934856371328/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.740106:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177375.740109:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177375.740111:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab3400.
02000000:00000001:2.0:1786177375.740114:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177375.740117:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177375.740119:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:2.0:1786177375.740121:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177375.740123:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6000.
02000000:00000001:2.0:1786177375.740127:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177375.740128:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177375.740130:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba419c00.
00000100:00000001:2.0:1786177375.740134:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177375.740135:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.740137:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177375.740140:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba41bb80 x1872934856371456/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.740146:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177375.740148:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177375.740149:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab0800.
02000000:00000001:2.0:1786177375.740152:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177375.740153:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177375.740155:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177375.740156:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177375.740158:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a6800.
02000000:00000001:2.0:1786177375.740160:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177375.740161:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177375.740162:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba41bb80.
00000100:00000001:2.0:1786177375.740165:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177375.740166:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.740168:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1786177375.740171:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba418700 x1872934856371584/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177375.740178:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1786177375.740180:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1786177375.740181:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186ab2400.
02000000:00000001:2.0:1786177375.740184:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1786177375.740185:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:2.0:1786177375.740186:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:2.0:1786177375.740188:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:2.0:1786177375.740189:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1885a4000.
02000000:00000001:2.0:1786177375.740191:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1786177375.740192:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1786177375.740194:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba418700.
00000100:00000001:2.0:1786177375.740196:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1786177375.740197:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.740200:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1786177375.740207:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1786177375.740209:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177375.740212:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1786177375.740213:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177376.313672:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177376.313681:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177376.313688:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177376.313689:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177376.313693:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177376.313694:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177376.313797:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177376.313802:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177376.313805:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177376.760698:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177376.760709:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177376.760712:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177376.760721:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177376.760726:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177376.760731:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177376.760734:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177376.760737:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177376.760740:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177377.336683:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177377.336692:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177377.336700:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177377.336701:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177377.336706:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177377.336707:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177377.340544:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177377.340553:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177377.340558:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177377.786488:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177377.786496:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177377.786498:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177377.786503:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177377.786507:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177377.786530:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177377.786534:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177377.786536:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177377.786539:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177378.360681:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177378.360691:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177378.360699:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177378.360700:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177378.360703:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177378.360704:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177378.360797:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177378.360803:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177378.360807:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177378.808661:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177378.808672:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177378.808675:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177378.808684:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177378.808689:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177378.808693:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177378.808697:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177378.808700:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177378.808703:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177379.385845:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177379.385854:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177379.385860:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177379.385862:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177379.385865:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177379.385866:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177379.391079:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177379.391089:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177379.391094:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:1.0:1786177379.576661:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302275278, at 4302275328
00000100:00000010:1.0:1786177379.576680:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f254e0.
00000100:00000040:0.0:1786177379.576828:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302275278, at 4302275328
00000100:00000010:0.0:1786177379.576835:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1b9f25960.
00000100:00000001:3.0:1786177379.835119:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177379.835128:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177379.835130:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177379.835137:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177379.835141:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177379.835146:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177379.835149:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177379.835153:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177379.835156:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177380.408685:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177380.408693:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177380.408700:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177380.408701:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177380.408705:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177380.408706:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177380.408802:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177380.408808:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177380.408812:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786177380.856996:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177380.857003:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177380.857006:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177380.857009:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177380.857014:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177380.857022:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7604
00000400:00000200:2.0:1786177380.857030:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177380.857037:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177380.857040:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177380.857044:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177380.857047:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177380.857049:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177380.857052:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177380.857055:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177380.857057:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177380.857061:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177380.857064:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177380.857066:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177380.857068:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177380.857071:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177380.857072:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177380.857075:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177380.857077:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177380.857079:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177380.857082:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177380.857084:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177380.857087:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177380.857090:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177380.857092:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177380.857094:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177380.857096:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177380.857102:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edfb80.
00000020:00000040:2.0:1786177380.857105:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177380.857109:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177380.857112:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177380.857114:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177380.857118:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177380.857121:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177380.857361:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000001:2.0:1786177380.857369:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177380.857379:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177380.857389:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177380.857394:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177380.857399:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041cc0335] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.857465:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177380.857468:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177380.857471:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177380.857478:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.857486:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177380.857488:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177380.857491:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177380.857494:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177380.857496:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177380.857500:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856371712:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177380.857503:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177380.857507:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177380.857530:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177380.857533:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177380.857538:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786177380.857540:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177380.857545:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177380.857549:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177380.857556:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177380.857562:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856371712, portal 25
00000100:00000001:1.0:1786177380.857564:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177380.857566:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454191488 : -114548255360128 : ffff97d1a7edfb80)
00000100:00000040:1.0:1786177380.857571:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.857578:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177380.857581:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177380.857594:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177380.857597:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856371712, offset 0
00000400:00000200:1.0:1786177380.857604:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177380.857611:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177380.857616:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177380.857621:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177380.857625:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177380.857628:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177380.857631:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177380.857634:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255075:255075:256:4294967295] 192.168.202.110@tcp LPNI seq info [255064:255064:8:4294967295]
00000400:00000200:1.0:1786177380.857638:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177380.857647:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177380.857654:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177380.857658:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177380.857664:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177380.857667:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786177380.857672:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177380.857675:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786177380.857679:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177380.857681:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177380.857686:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177380.857691:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177380.857704:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177380.857706:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177380.857710:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177380.857714:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177380.857799:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177380.857803:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177380.857807:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177380.857808:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177380.857809:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177380.857812:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177380.857815:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7604
00000400:00000200:2.0:1786177380.857819:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177380.857822:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177380.857824:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177380.857827:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177380.857830:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177380.857832:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177380.857834:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177380.857836:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177380.857838:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177380.857840:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177380.857842:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177380.857844:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177380.857845:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177380.857848:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177380.857850:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177380.857851:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177380.857853:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177380.857855:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177380.857858:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177380.857861:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177380.857863:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177380.857867:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177380.857869:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177380.857870:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177380.857874:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177380.857877:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177380.857879:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177380.857883:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edea00.
00000020:00000040:2.0:1786177380.857885:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177380.857888:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177380.857889:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177380.857891:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177380.857892:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177380.857894:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177380.857898:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786177380.857901:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177380.857906:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177380.857914:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177380.857916:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177380.857919:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ac2e9178] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1786177380.858877:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177380.858882:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177380.858886:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177380.858890:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177380.858893:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177380.858896:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177380.858899:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177380.858901:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177380.858904:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.867238:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177380.867255:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.867267:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.867269:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177380.867282:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177380.867290:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177380.867293:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302280407
00000100:00000010:1.0:1786177380.867302:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b17e0.
00000100:00000001:1.0:1786177380.867308:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1786177380.867314:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302280407
00000100:00000010:1.0:1786177380.867316:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b19c0.
00000100:00000001:1.0:1786177380.867319:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1786177380.867323:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177380.867326:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.867329:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177380.867333:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177380/real 1786177380]  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177380.867345:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177380.867347:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177380.867350:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177380.867353:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177380.867356:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177380.867359:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177380.867362:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177380.867371:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.867377:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177380.867379:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.867381:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177380.867383:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177380.867387:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.867394:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177380.867398:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.867406:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177380.867409:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.867411:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177380.867412:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.867415:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177380.867417:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177380.867420:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177380.867425:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177380.867427:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177380.867431:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177380.867439:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856371712:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177380.867444:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.867446:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177380.867449:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edfb80 x1872934856371712/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.867456:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177380.867459:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177380.867462:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786177380.867466:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177380.867468:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177380.867470:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786177380.867472:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177380.867475:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3400.
02000000:00000001:1.0:1786177380.867479:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177380.867480:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177380.867482:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edfb80.
00000100:00000001:1.0:1786177380.867485:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177380.867487:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.867489:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.867495:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177380.867497:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.867500:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177380.867502:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177380.867503:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177380.867526:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.867534:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177380.867536:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177380.867539:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177380.867542:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177380.867545:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177380.867548:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856371840:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177380.867552:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177380.867555:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177380.867557:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177380.867560:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177380.867564:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786177380.867567:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177380.867571:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177380.867575:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177380.867579:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177380.867879:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856371840, portal 10
00000100:00000001:1.0:1786177380.867883:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177380.867885:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454187008 : -114548255364608 : ffff97d1a7edea00)
00000100:00000040:1.0:1786177380.867893:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.867900:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177380.867902:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177380.867905:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177380.867909:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856371840, offset 0
00000400:00000200:1.0:1786177380.867915:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177380.867922:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177380.867928:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177380.867933:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177380.867943:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177380.867947:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177380.867949:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177380.867953:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255076:255076:256:4294967295] 192.168.202.110@tcp LPNI seq info [255065:255065:8:4294967295]
00000400:00000200:1.0:1786177380.867957:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177380.867966:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177380.867972:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177380.867977:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177380.867983:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177380.867987:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786177380.868002:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177380.868005:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786177380.868009:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177380.868012:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177380.868017:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177380.868021:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177380.868024:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177380.868026:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177380.868029:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177380.868033:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.868040:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177380.868043:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.868050:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.868052:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177380.868055:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177380.868058:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786177380.868061:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177380.868063:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302276628
00000100:00000001:1.0:1786177380.868065:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302280407
00000100:00000001:1.0:1786177380.868067:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302276628
00000100:00000001:1.0:1786177380.868069:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302280407
00000100:00000001:1.0:1786177380.868071:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177380.868073:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.868075:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177380.868078:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177380/real 1786177380]  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177380.868086:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177380.868087:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177380.868089:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177380.868091:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177380.868093:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177380.868095:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177380.868098:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177380.868107:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.868113:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177380.868114:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.868115:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177380.868117:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177380.868120:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.868666:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177380.868672:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.868680:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177380.868683:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.868684:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177380.868686:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.868687:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177380.868689:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177380.868692:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177380.868700:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177380.868702:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177380.868707:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177380.868715:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856371840:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177380.868718:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.868721:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177380.868723:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edea00 x1872934856371840/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.868730:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177380.868732:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177380.868735:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786177380.868738:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177380.868739:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177380.868741:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177380.868743:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177380.868745:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:1.0:1786177380.868747:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177380.868748:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177380.868750:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edea00.
00000100:00000001:1.0:1786177380.868753:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177380.868754:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.868756:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.868760:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177380.868761:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.868763:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177380.868765:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177380.870204:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177380.870211:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177380.870215:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177380.870217:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177380.870219:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177380.870222:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177380.870227:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7604
00000400:00000200:2.0:1786177380.870234:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177380.870238:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177380.870241:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177380.870245:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177380.870248:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177380.870250:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177380.870253:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177380.870256:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177380.870258:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177380.870260:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177380.870262:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177380.870265:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786177380.870267:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786177380.870270:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177380.870272:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177380.870274:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177380.870275:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177380.870278:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786177380.870280:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177380.870283:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177380.870286:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177380.870459:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177380.870462:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177380.870465:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177380.870468:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177380.870471:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177380.870473:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177380.870481:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d500.
00000020:00000040:3.0:1786177380.870484:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177380.870488:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177380.870490:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177380.870493:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177380.870495:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177380.870499:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177380.870505:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fbc00.
00000100:00000001:3.0:1786177380.870531:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177380.870538:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177380.870547:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177380.870551:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177380.870555:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7c36393] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177380.870565:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177380.870567:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177380.870570:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177380.870572:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177380.870573:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177380.870576:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177380.870578:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7604
00000400:00000200:3.0:1786177380.870590:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177380.870594:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177380.870596:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177380.870599:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177380.870601:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177380.870603:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177380.870605:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177380.870608:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177380.870609:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177380.870610:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177380.870612:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177380.870614:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177380.870616:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177380.870619:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177380.870621:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177380.870622:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177380.870624:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177380.870626:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177380.870629:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177380.870631:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177380.870632:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177380.870635:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177380.870636:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177380.870638:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177380.870640:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177380.870642:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177380.870644:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177380.870647:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69ea00.
00000020:00000040:3.0:1786177380.870649:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177380.870651:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177380.870653:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177380.870654:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177380.870655:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177380.870657:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177380.870661:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fa400.
00000100:00000001:3.0:1786177380.870664:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177380.870668:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177380.870675:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177380.870677:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177380.870680:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053162887] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177380.870687:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177380.870690:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177380.870693:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177380.870694:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177380.870695:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177380.870697:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177380.870699:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7604
00000400:00000200:3.0:1786177380.870702:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177380.870705:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177380.870707:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177380.870710:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177380.870711:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177380.870713:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177380.870715:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177380.870717:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177380.870718:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177380.870719:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177380.870721:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177380.870723:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177380.870725:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177380.870727:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177380.870729:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177380.870731:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177380.870733:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177380.870736:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786177380.870738:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177380.870740:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177380.870741:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177380.870743:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177380.870745:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177380.870746:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177380.870748:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177380.870750:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177380.870751:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177380.870754:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c000.
00000020:00000040:3.0:1786177380.870756:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177380.870758:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177380.870759:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177380.870761:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177380.870762:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177380.870764:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177380.870768:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f8000.
00000100:00000001:3.0:1786177380.870771:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177380.870775:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177380.870781:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177380.870783:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177380.870786:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006fe8fdb7] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177380.870793:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177380.870798:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7614)
00000100:00000001:1.0:1786177380.875262:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177380.875271:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177380.875273:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177380.875291:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875302:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177380.875305:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177380.875309:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177380.875313:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177380.875315:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177380.875318:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856371968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177380.875322:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177380.875327:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177380.875329:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177380.875331:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177380.875341:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786177380.875343:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177380.875351:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177380.875356:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177380.875363:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177380.875369:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856371968, portal 10
00000100:00000001:1.0:1786177380.875371:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177380.875373:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297984 : -114547945253632 : ffff97d1ba69d500)
00000100:00000040:1.0:1786177380.875380:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875388:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177380.875391:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177380.875393:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177380.875395:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856371968, offset 0
00000400:00000200:1.0:1786177380.875401:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177380.875409:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177380.875416:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177380.875421:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177380.875425:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177380.875428:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177380.875430:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177380.875435:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255077:255077:256:4294967295] 192.168.202.110@tcp LPNI seq info [255066:255066:8:4294967295]
00000400:00000200:1.0:1786177380.875439:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177380.875448:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177380.875454:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177380.875460:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177380.875466:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177380.875469:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786177380.875475:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177380.875478:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786177380.875482:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177380.875485:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177380.875491:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177380.875495:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177380.875499:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177380.875502:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177380.875506:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177380.875555:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875565:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177380.875568:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875576:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.875578:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177380.875584:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177380.875597:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177380.875601:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177380.875604:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302276760
00000100:00000001:1.0:1786177380.875607:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302280407
00000100:00000001:1.0:1786177380.875609:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302276760
00000100:00000001:1.0:1786177380.875611:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302280407
00000100:00000001:1.0:1786177380.875614:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177380.875616:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.875619:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177380.875622:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177380/real 1786177380]  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177380.875639:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177380.875640:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177380.875642:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177380.875645:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177380.875646:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177380.875649:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177380.875652:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177380.875661:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875667:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177380.875668:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.875671:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177380.875673:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177380.875676:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875682:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177380.875687:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875695:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177380.875697:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.875699:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177380.875700:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.875703:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177380.875705:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177380.875708:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177380.875712:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177380.875714:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177380.875719:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177380.875727:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856371968:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177380.875731:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177380.875735:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875741:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177380.875742:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177380.875745:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177380.875747:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177380.875749:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177380.875751:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856372096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177380.875754:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177380.875756:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177380.875758:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177380.875759:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177380.875762:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177380.875764:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177380.875767:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177380.875770:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177380.875773:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177380.875776:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856372096, portal 4
00000100:00000001:1.0:1786177380.875778:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177380.875779:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764303360 : -114547945248256 : ffff97d1ba69ea00)
00000100:00000040:1.0:1786177380.875784:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875790:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177380.875793:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177380.875794:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177380.875796:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856372096, offset 0
00000400:00000200:1.0:1786177380.875801:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177380.875804:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177380.875807:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177380.875810:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177380.875812:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177380.875814:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177380.875816:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177380.875819:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255078:255078:256:4294967295] 192.168.202.110@tcp LPNI seq info [255067:255067:8:4294967295]
00000400:00000200:1.0:1786177380.875823:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177380.875829:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177380.875834:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177380.875837:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177380.875841:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177380.875843:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786177380.875847:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177380.875850:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786177380.875854:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177380.875856:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177380.875859:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177380.875862:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177380.875865:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177380.875867:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177380.875869:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177380.875872:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875879:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177380.875883:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875889:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.875890:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177380.875893:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177380.875896:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177380.875899:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177380.875901:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302276635
00000100:00000001:1.0:1786177380.875903:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302280407
00000100:00000001:1.0:1786177380.875905:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302276635
00000100:00000001:1.0:1786177380.875907:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302280407
00000100:00000001:1.0:1786177380.875909:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177380.875911:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.875913:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177380.875917:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177380/real 1786177380]  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177380.875923:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177380.875925:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177380.875926:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177380.875928:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177380.875930:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177380.875932:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177380.875936:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177380.875944:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875949:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177380.875950:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.875952:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177380.875953:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177380.875957:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875963:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177380.875966:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.875973:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177380.875975:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.875977:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177380.875978:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.875979:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177380.875981:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177380.875983:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177380.875986:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177380.875999:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177380.876003:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177380.876011:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856372096:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177380.876015:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177380.876018:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.876025:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177380.876026:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177380.876029:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177380.876030:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177380.876032:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177380.876035:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856372224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177380.876037:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177380.876039:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177380.876041:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177380.876043:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177380.876046:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177380.876047:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177380.876050:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177380.876052:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177380.876055:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177380.876058:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856372224, portal 4
00000100:00000001:1.0:1786177380.876061:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177380.876062:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764292608 : -114547945259008 : ffff97d1ba69c000)
00000100:00000040:1.0:1786177380.876067:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.876074:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177380.876076:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177380.876077:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177380.876079:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856372224, offset 0
00000400:00000200:1.0:1786177380.876083:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177380.876086:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177380.876089:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177380.876092:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177380.876094:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177380.876096:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177380.876098:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177380.876101:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255079:255079:256:4294967295] 192.168.202.110@tcp LPNI seq info [255068:255068:8:4294967295]
00000400:00000200:1.0:1786177380.876106:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177380.876111:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177380.876115:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177380.876118:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177380.876122:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177380.876124:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1786177380.876128:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177380.876130:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1786177380.876134:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177380.876136:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177380.876139:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177380.876142:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177380.876145:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177380.876147:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177380.876149:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177380.876151:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.876165:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177380.876167:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.876174:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.876175:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177380.876177:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177380.876180:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177380.876183:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177380.876185:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302276643
00000100:00000001:1.0:1786177380.876187:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302280407
00000100:00000001:1.0:1786177380.876189:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302276643
00000100:00000001:1.0:1786177380.876191:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302280407
00000100:00000001:1.0:1786177380.876193:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177380.876195:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.876197:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177380.876200:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177380/real 1786177380]  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177380.876207:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177380.876208:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177380.876209:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177380.876211:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177380.876212:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177380.876214:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177380.876217:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177380.876225:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.876231:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177380.876233:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.876235:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177380.876236:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177380.876239:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.876245:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177380.876249:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.876256:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177380.876258:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.876259:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177380.876261:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.876262:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177380.876264:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177380.876266:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177380.876269:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177380.876271:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177380.876275:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177380.876283:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856372224:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177380.876286:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.876288:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177380.876292:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d500 x1872934856371968/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.876298:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177380.876301:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177380.876303:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b03400.
02000000:00000001:1.0:1786177380.876306:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177380.876308:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177380.876310:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177380.876312:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177380.876314:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fbc00.
02000000:00000001:1.0:1786177380.876317:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177380.876319:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177380.876321:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d500.
00000100:00000001:1.0:1786177380.876324:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177380.876326:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.876327:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177380.876330:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69ea00 x1872934856372096/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.876337:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177380.876338:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177380.876339:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177380.876341:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177380.876342:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177380.876343:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177380.876344:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177380.876345:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fa400.
02000000:00000001:1.0:1786177380.876348:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177380.876349:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177380.876350:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69ea00.
00000100:00000001:1.0:1786177380.876352:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177380.876354:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.876355:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177380.876358:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c000 x1872934856372224/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177380.876363:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177380.876364:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177380.876366:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177380.876367:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177380.876368:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177380.876370:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177380.876372:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177380.876373:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f8000.
02000000:00000001:1.0:1786177380.876375:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177380.876376:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177380.876377:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c000.
00000100:00000001:1.0:1786177380.876379:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177380.876381:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.876383:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177380.876390:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177380.876391:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177380.876394:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177380.876395:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177381.432645:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177381.432653:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177381.432659:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177381.432660:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177381.432664:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177381.432665:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177381.432798:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177381.432802:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177381.432806:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177381.880610:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177381.880619:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177381.880621:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177381.880627:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177381.880630:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177381.880634:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177381.880637:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177381.880640:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177381.880642:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177382.456639:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177382.456649:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177382.456651:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177382.456653:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177382.456657:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177382.456660:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177382.456661:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177382.456664:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177382.456667:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177382.456668:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177382.456669:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177382.456671:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177382.456672:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177382.456674:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177382.456675:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177382.456677:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177382.456688:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177382.456689:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177382.459789:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177382.459796:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177382.459800:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177382.904640:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177382.904647:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177382.904649:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177382.904656:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177382.904660:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177382.904663:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177382.904666:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177382.904669:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177382.904671:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177383.480631:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177383.480637:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177383.480644:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177383.480645:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177383.480648:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177383.480650:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177383.481012:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177383.481016:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177383.481019:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177383.928753:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177383.928760:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177383.928762:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177383.928767:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177383.928771:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177383.928775:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177383.928778:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177383.928780:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177383.928783:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177384.504640:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177384.504648:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177384.504654:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177384.504655:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177384.504659:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177384.504660:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177384.505001:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177384.505005:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177384.505008:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177384.696640:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302280407, at 4302280448
00000100:00000040:1.0:1786177384.696640:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302280407, at 4302280448
00000100:00000010:1.0:1786177384.696656:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b19c0.
00000100:00000010:0.0:1786177384.700034:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b17e0.
00000100:00000001:0.0:1786177384.953069:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177384.953076:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177384.953078:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177384.953084:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177384.953088:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177384.953092:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177384.953094:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177384.953097:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177384.953100:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177385.529206:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177385.529212:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177385.529217:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177385.529219:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177385.529221:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177385.529225:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177385.529227:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177385.529230:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177385.529232:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177385.529235:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177385.529236:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177385.529239:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177385.529291:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177385.529296:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177385.529299:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177385.976611:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177385.976618:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177385.976620:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177385.976624:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177385.976627:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177385.976630:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177385.976632:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177385.976633:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177385.976635:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177385.977247:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177385.977255:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1786177385.977257:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177385.977260:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177385.977264:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177385.977271:0:24919:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7609
00000400:00000200:3.0:1786177385.977281:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177385.977288:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177385.977291:0:24919:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177385.977294:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.977298:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177385.977300:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.977302:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177385.977306:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177385.977308:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:3.0:1786177385.977310:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177385.977314:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:3.0:1786177385.977316:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:3.0:1786177385.977318:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:3.0:1786177385.977321:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.977323:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177385.977325:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.977327:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177385.977330:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:3.0:1786177385.977333:0:24919:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177385.977335:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177385.977339:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177385.977342:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177385.977345:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.977347:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:3.0:1786177385.977350:0:24919:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177385.977359:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c700.
00000020:00000040:3.0:1786177385.977362:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:3.0:1786177385.977366:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177385.977368:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.977371:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177385.977373:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177385.977454:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177385.977464:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9000.
00000100:00000001:3.0:1786177385.977471:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177385.977481:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177385.977491:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177385.977496:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177385.977501:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e56458ae] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.977588:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177385.977592:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177385.977594:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177385.977600:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.977608:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177385.977609:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177385.977613:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177385.977615:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177385.977618:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177385.977621:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856372352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177385.977626:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177385.977629:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177385.977632:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177385.977635:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177385.977640:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177385.977642:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177385.977648:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177385.977652:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177385.977658:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177385.977665:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856372352, portal 25
00000100:00000001:1.0:1786177385.977668:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177385.977670:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:1.0:1786177385.977675:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.977683:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177385.977686:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177385.977688:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177385.977691:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856372352, offset 0
00000400:00000200:1.0:1786177385.977698:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177385.977706:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177385.977711:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177385.977716:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177385.977719:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177385.977723:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177385.977726:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177385.977729:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255080:255080:256:4294967295] 192.168.202.110@tcp LPNI seq info [255069:255069:8:4294967295]
00000400:00000200:1.0:1786177385.977734:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177385.977743:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177385.977749:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177385.977754:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177385.977761:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177385.977765:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786177385.977781:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177385.977785:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786177385.977789:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177385.977806:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177385.977813:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177385.977818:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177385.977823:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177385.977826:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177385.977830:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177385.977834:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177385.978983:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177385.978988:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177385.978991:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1786177385.978992:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177385.978995:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177385.978998:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177385.979001:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7609
00000400:00000200:3.0:1786177385.979006:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177385.979009:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177385.979016:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177385.979019:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.979022:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177385.979024:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.979026:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177385.979029:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177385.979030:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:3.0:1786177385.979033:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177385.979035:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:3.0:1786177385.979037:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:3.0:1786177385.979039:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:3.0:1786177385.979042:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.979044:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177385.979049:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.979052:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177385.979055:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:3.0:1786177385.979057:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177385.979060:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177385.979062:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177385.979067:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177385.979070:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.979072:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177385.979076:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177385.979079:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.979081:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177385.979086:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e300.
00000020:00000040:3.0:1786177385.979089:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177385.979092:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177385.979093:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.979095:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177385.979097:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177385.979099:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177385.979104:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6f9c00.
00000100:00000001:3.0:1786177385.979109:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177385.979114:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177385.979123:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177385.979126:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177385.979129:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e3752a60] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.979551:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177385.979558:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.979568:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.979569:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177385.979578:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177385.979583:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177385.979586:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302285519
00000100:00000010:1.0:1786177385.979591:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b19c0.
00000100:00000001:1.0:1786177385.979597:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1786177385.979605:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302285519
00000100:00000010:1.0:1786177385.979608:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1360.
00000100:00000001:1.0:1786177385.979611:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1786177385.979614:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177385.979617:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.979620:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177385.979624:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177385/real 1786177385]  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177385.979635:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177385.979636:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177385.979639:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177385.979642:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177385.979644:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177385.979647:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177385.979650:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177385.979659:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.979667:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177385.979668:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.979671:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177385.979672:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177385.979677:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.979684:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177385.979688:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.979695:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177385.979698:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.979700:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177385.979701:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.979703:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177385.979705:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177385.979708:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177385.979711:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177385.979714:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177385.979718:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177385.979726:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856372352:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177385.979731:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.979733:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177385.979736:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856372352/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.979748:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177385.979751:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177385.979754:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177385.979757:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177385.979759:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177385.979761:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786177385.979763:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177385.979766:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9000.
02000000:00000001:1.0:1786177385.979769:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177385.979771:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177385.979773:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:1.0:1786177385.979776:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177385.979777:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.979779:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.979786:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177385.979788:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.979791:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177385.979792:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177385.979794:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177385.979798:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.979805:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177385.979806:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177385.979809:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177385.979810:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177385.979813:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177385.979828:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856372480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177385.979831:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177385.979833:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177385.979835:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177385.979837:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177385.979840:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177385.979842:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177385.979846:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177385.979849:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177385.979853:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177385.979857:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856372480, portal 10
00000100:00000001:1.0:1786177385.979859:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177385.979860:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:1.0:1786177385.979867:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.979873:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177385.979875:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177385.979878:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177385.979880:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856372480, offset 0
00000400:00000200:1.0:1786177385.979885:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177385.979890:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177385.979895:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177385.979899:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177385.979902:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177385.979906:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177385.979918:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177385.979922:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255081:255081:256:4294967295] 192.168.202.110@tcp LPNI seq info [255070:255070:8:4294967295]
00000400:00000200:1.0:1786177385.979927:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177385.979934:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177385.979950:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177385.979954:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177385.979959:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177385.979962:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786177385.979970:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177385.979973:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786177385.979978:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177385.979980:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177385.979983:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177385.979987:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177385.979991:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177385.979992:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177385.979995:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177385.979999:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.980007:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177385.980010:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.980016:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.980018:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177385.980021:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177385.980025:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786177385.980028:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177385.980030:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302281739
00000100:00000001:1.0:1786177385.980033:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302285519
00000100:00000001:1.0:1786177385.980035:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302281739
00000100:00000001:1.0:1786177385.980037:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302285519
00000100:00000001:1.0:1786177385.980039:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177385.980041:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.980043:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177385.980051:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177385/real 1786177385]  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177385.980059:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177385.980060:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177385.980062:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177385.980064:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177385.980066:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177385.980068:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177385.980071:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177385.980078:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.980090:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177385.980091:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.980093:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177385.980094:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177385.980097:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.980113:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177385.980117:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.980124:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177385.980126:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.980127:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177385.980129:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.980130:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177385.980132:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177385.980134:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177385.980137:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177385.980140:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177385.980144:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177385.980151:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856372480:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177385.980155:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.980163:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177385.980166:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856372480/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.980172:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177385.980174:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177385.980176:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177385.980248:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177385.980250:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177385.980251:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177385.980254:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177385.980256:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6f9c00.
02000000:00000001:1.0:1786177385.980259:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177385.980260:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177385.980262:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:1.0:1786177385.980264:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177385.980266:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.980268:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.980271:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177385.980273:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.980274:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177385.980276:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.980349:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177385.980352:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.980385:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177385.980389:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177385.980392:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1786177385.980394:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177385.980401:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177385.980404:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177385.980407:0:24919:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7609
00000400:00000200:3.0:1786177385.980412:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177385.980415:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177385.980418:0:24919:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177385.980420:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.980422:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177385.980425:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.980427:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177385.980429:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177385.980430:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:3.0:1786177385.980432:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177385.980434:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:3.0:1786177385.980436:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:3.0:1786177385.980438:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:3.0:1786177385.980441:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.980443:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177385.980445:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.980447:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177385.980449:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:3.0:1786177385.980452:0:24919:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177385.980454:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177385.980456:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177385.980459:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177385.980461:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.980463:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177385.980466:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1786177385.980468:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.980470:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177385.980474:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69d180.
00000020:00000040:3.0:1786177385.980476:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:3.0:1786177385.980479:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177385.980480:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.980482:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177385.980484:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177385.980486:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177385.980490:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb000.
00000100:00000001:3.0:1786177385.980494:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177385.980499:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177385.980505:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177385.980507:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177385.980528:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000085df314e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.980931:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177385.980934:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177385.980935:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177385.980955:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.980963:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177385.980964:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177385.980966:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177385.980968:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177385.980970:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177385.980972:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856372608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177385.980993:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177385.980995:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177385.980996:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177385.980998:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177385.981001:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177385.981003:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177385.981008:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177385.981010:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177385.981013:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177385.981017:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856372608, portal 10
00000100:00000001:1.0:1786177385.981018:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177385.981020:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764297088 : -114547945254528 : ffff97d1ba69d180)
00000100:00000040:1.0:1786177385.981026:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.981032:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177385.981034:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177385.981037:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177385.981039:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856372608, offset 0
00000400:00000200:1.0:1786177385.981043:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177385.981048:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177385.981052:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177385.981055:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177385.981058:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177385.981060:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177385.981063:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177385.981066:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255082:255082:256:4294967295] 192.168.202.110@tcp LPNI seq info [255071:255071:8:4294967295]
00000400:00000200:1.0:1786177385.981069:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177385.981075:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177385.981079:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177385.981082:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177385.981086:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177385.981089:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786177385.981093:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177385.981096:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786177385.981099:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177385.981101:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177385.981104:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177385.981108:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177385.981111:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177385.981113:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177385.981116:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177385.981119:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177385.982311:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177385.982315:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177385.982318:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:3.0:1786177385.982319:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177385.982321:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177385.982323:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177385.982327:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7609
00000400:00000200:3.0:1786177385.982330:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177385.982333:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177385.982335:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177385.982337:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.982338:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177385.982340:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.982341:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177385.982343:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177385.982344:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:3.0:1786177385.982345:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177385.982347:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:3.0:1786177385.982349:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:3.0:1786177385.982350:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:3.0:1786177385.982351:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.982353:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177385.982354:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.982355:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177385.982357:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:3.0:1786177385.982358:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177385.982359:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177385.982361:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177385.982364:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177385.982365:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.982366:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177385.982368:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177385.982370:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.982371:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177385.982375:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69e300.
00000020:00000040:3.0:1786177385.982377:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177385.982379:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177385.982380:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.982381:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177385.982383:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177385.982385:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177385.982389:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786177385.982392:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177385.982396:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177385.982401:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177385.982403:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177385.982406:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e3752a60] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983151:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177385.983158:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983166:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.983167:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177385.983175:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177385.983188:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177385.983192:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177385.983194:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302281867
00000100:00000001:1.0:1786177385.983196:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302285519
00000100:00000001:1.0:1786177385.983197:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302281867
00000100:00000001:1.0:1786177385.983199:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302285519
00000100:00000001:1.0:1786177385.983200:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177385.983202:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.983203:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177385.983206:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177385/real 1786177385]  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177385.983213:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177385.983214:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177385.983217:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177385.983219:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177385.983220:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177385.983222:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177385.983225:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177385.983230:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983234:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177385.983236:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.983237:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177385.983238:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177385.983240:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983244:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177385.983247:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983251:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177385.983252:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.983253:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177385.983254:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.983255:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177385.983256:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177385.983258:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177385.983260:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177385.983261:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177385.983263:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177385.983268:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856372608:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177385.983270:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.983271:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177385.983273:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69d180 x1872934856372608/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983277:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177385.983279:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177385.983281:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177385.983283:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177385.983284:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177385.983285:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177385.983286:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177385.983288:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb000.
02000000:00000001:1.0:1786177385.983290:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177385.983291:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177385.983292:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69d180.
00000100:00000001:1.0:1786177385.983294:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177385.983295:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.983296:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.983300:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177385.983301:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.983302:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177385.983303:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177385.983305:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177385.983307:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983311:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177385.983312:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177385.983314:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177385.983315:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177385.983316:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177385.983318:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856372736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177385.983320:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177385.983321:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177385.983323:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177385.983324:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177385.983326:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177385.983327:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177385.983330:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177385.983332:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177385.983334:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177385.983337:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856372736, portal 4
00000100:00000001:1.0:1786177385.983338:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177385.983339:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764301568 : -114547945250048 : ffff97d1ba69e300)
00000100:00000040:1.0:1786177385.983343:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983347:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177385.983348:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177385.983350:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177385.983351:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856372736, offset 0
00000400:00000200:1.0:1786177385.983354:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177385.983357:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177385.983360:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177385.983363:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177385.983365:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177385.983368:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177385.983369:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177385.983372:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255083:255083:256:4294967295] 192.168.202.110@tcp LPNI seq info [255072:255072:8:4294967295]
00000400:00000200:1.0:1786177385.983374:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177385.983379:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177385.983383:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177385.983385:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177385.983388:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177385.983390:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786177385.983393:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177385.983395:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786177385.983398:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177385.983399:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177385.983401:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177385.983404:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177385.983406:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177385.983407:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177385.983408:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177385.983411:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983419:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177385.983421:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983425:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.983426:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177385.983427:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177385.983429:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177385.983430:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177385.983431:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302281742
00000100:00000001:1.0:1786177385.983433:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302285519
00000100:00000001:1.0:1786177385.983434:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302281742
00000100:00000001:1.0:1786177385.983435:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302285519
00000100:00000001:1.0:1786177385.983437:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177385.983438:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.983439:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177385.983441:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177385/real 1786177385]  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177385.983444:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177385.983445:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177385.983446:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177385.983447:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177385.983448:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177385.983449:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177385.983451:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177385.983456:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983459:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177385.983460:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.983461:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177385.983462:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177385.983464:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983468:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177385.983470:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983473:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177385.983474:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.983475:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177385.983476:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.983477:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177385.983477:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177385.983479:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177385.983480:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177385.983482:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177385.983483:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177385.983487:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856372736:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177385.983489:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.983490:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177385.983492:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69e300 x1872934856372736/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.983496:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177385.983497:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177385.983498:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177385.983499:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177385.983500:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177385.983501:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177385.983502:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177385.983503:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:1.0:1786177385.983504:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177385.983505:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177385.983506:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69e300.
00000100:00000001:1.0:1786177385.983507:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177385.983508:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.983509:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.983522:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177385.983523:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.983524:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177385.983526:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.984236:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177385.984238:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.984266:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1786177385.984269:0:24919:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1786177385.984271:0:24919:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1786177385.984272:0:24919:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1786177385.984274:0:24919:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1786177385.984276:0:24919:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1786177385.984279:0:24919:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7609
00000400:00000200:3.0:1786177385.984282:0:24919:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:3.0:1786177385.984284:0:24919:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1786177385.984286:0:24919:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1786177385.984287:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.984289:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177385.984291:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.984292:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:3.0:1786177385.984294:0:24919:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:3.0:1786177385.984294:0:24919:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:3.0:1786177385.984296:0:24919:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:3.0:1786177385.984297:0:24919:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:3.0:1786177385.984299:0:24919:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:3.0:1786177385.984300:0:24919:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:3.0:1786177385.984302:0:24919:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.984303:0:24919:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1786177385.984305:0:24919:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:3.0:1786177385.984306:0:24919:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:3.0:1786177385.984307:0:24919:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:3.0:1786177385.984309:0:24919:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:3.0:1786177385.984310:0:24919:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1786177385.984311:0:24919:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1786177385.984313:0:24919:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1786177385.984314:0:24919:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.984316:0:24919:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:3.0:1786177385.984317:0:24919:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1786177385.984319:0:24919:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.984320:0:24919:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1786177385.984324:0:24919:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1ba69c700.
00000020:00000040:3.0:1786177385.984326:0:24919:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:3.0:1786177385.984327:0:24919:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1786177385.984328:0:24919:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177385.984330:0:24919:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1786177385.984331:0:24919:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1786177385.984333:0:24919:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1786177385.984336:0:24919:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d1ba6fb400.
00000100:00000001:3.0:1786177385.984338:0:24919:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177385.984342:0:24919:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1786177385.984346:0:24919:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1786177385.984348:0:24919:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177385.984351:0:24919:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e56458ae] to pc [ptlrpcd_rcv+-1]  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.984379:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177385.984381:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177385.984382:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177385.984385:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.984389:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177385.984390:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177385.984392:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177385.984393:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177385.984394:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177385.984396:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856372864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177385.984398:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177385.984399:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177385.984400:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177385.984401:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177385.984403:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177385.984404:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177385.984407:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177385.984408:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177385.984410:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177385.984413:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856372864, portal 4
00000100:00000001:1.0:1786177385.984414:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177385.984415:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525764294400 : -114547945257216 : ffff97d1ba69c700)
00000100:00000040:1.0:1786177385.984419:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.984423:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177385.984424:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177385.984426:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177385.984427:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856372864, offset 0
00000400:00000200:1.0:1786177385.984429:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177385.984432:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177385.984434:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177385.984436:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177385.984438:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177385.984440:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177385.984441:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177385.984443:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255084:255084:256:4294967295] 192.168.202.110@tcp LPNI seq info [255073:255073:8:4294967295]
00000400:00000200:1.0:1786177385.984446:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177385.984450:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177385.984452:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177385.984454:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177385.984456:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177385.984458:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1786177385.984461:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177385.984463:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1786177385.984466:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177385.984467:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177385.984469:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177385.984472:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177385.984474:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177385.984475:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177385.984484:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177385.984486:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1786177385.985213:0:24919:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1786177385.985218:0:24919:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7619)
00000100:00000001:1.0:1786177385.985291:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177385.985296:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.985301:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.985302:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177385.985306:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177385.985310:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177385.985312:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177385.985314:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302281753
00000100:00000001:1.0:1786177385.985316:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302285519
00000100:00000001:1.0:1786177385.985317:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302281753
00000100:00000001:1.0:1786177385.985319:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302285519
00000100:00000001:1.0:1786177385.985320:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177385.985322:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.985323:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177385.985326:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177385/real 1786177385]  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177385.985330:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177385.985331:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177385.985332:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177385.985334:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177385.985335:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177385.985336:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177385.985338:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177385.985343:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.985347:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177385.985349:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.985350:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177385.985351:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177385.985353:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.985357:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177385.985359:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.985363:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177385.985364:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.985365:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177385.985366:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.985367:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177385.985368:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177385.985370:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177385.985372:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177385.985373:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177385.985375:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177385.985379:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1ba69c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856372864:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177385.985381:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.985382:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177385.985384:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1ba69c700 x1872934856372864/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177385.985388:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177385.985389:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177385.985390:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00c00.
02000000:00000001:1.0:1786177385.985392:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177385.985393:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177385.985394:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177385.985396:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177385.985397:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d1ba6fb400.
02000000:00000001:1.0:1786177385.985399:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177385.985400:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177385.985401:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1ba69c700.
00000100:00000001:1.0:1786177385.985403:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177385.985403:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.985405:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177385.985408:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177385.985409:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177385.985410:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177385.985411:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177386.552676:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177386.552683:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177386.552689:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177386.552690:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177386.552693:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177386.552694:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177386.552767:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177386.552772:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177386.552776:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1786177387.000648:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1786177387.000658:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1786177387.000661:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1786177387.000668:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:0.0:1786177387.000673:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1786177387.000677:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177387.000681:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1786177387.000684:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1786177387.000687:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177387.576641:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177387.576649:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177387.576655:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177387.576657:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177387.576660:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177387.576661:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177387.609690:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177387.609701:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177387.609707:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177388.185278:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177388.185289:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177388.185293:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177388.185301:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177388.185307:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177388.185311:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177388.185314:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177388.185317:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177388.185321:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177388.601103:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177388.601112:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177388.601118:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177388.601120:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177388.601123:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177388.601125:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177388.616322:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177388.616332:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177388.616337:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177389.240664:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177389.240674:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177389.240676:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177389.240684:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177389.240689:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177389.240693:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177389.240696:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177389.240700:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177389.240703:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177389.624671:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177389.624680:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177389.624688:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177389.624689:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177389.624692:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177389.624694:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1786177389.624864:0:103738:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1786177389.624869:0:103738:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1786177389.624872:0:103738:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:0.0:1786177389.816715:0:104866:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343000 (cpu 0) has reached its deadline 4302285519, at 4302285568
00000100:00000010:0.0:1786177389.816734:0:104866:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b19c0.
00000100:00000040:1.0:1786177389.816968:0:104921:0:(connection.c:75:cpu_latency_work()) work item of ffff97d183343110 (cpu 1) has reached its deadline 4302285519, at 4302285568
00000100:00000010:1.0:1786177389.816976:0:104921:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff97d1a79b1360.
00000100:00000001:3.0:1786177390.264660:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177390.264673:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177390.264676:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177390.264686:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177390.264691:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177390.264696:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177390.264699:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177390.264703:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177390.264706:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177390.648690:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177390.648699:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177390.648705:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177390.648707:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177390.648710:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177390.648711:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1786177390.650020:0:92301:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:3.0:1786177390.650026:0:92301:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:3.0:1786177390.650030:0:92301:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1786177391.096761:0:84480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 375c59b7-3e87-4aac-b0e2-3bc92f101731->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177391.096775:0:84480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1786177391.096778:0:84480:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177391.096784:0:84480:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177391.096791:0:84480:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177391.096799:0:84480:0:(import.c:510:import_select_connection()) MGC192.168.202.110@tcp: connect to NID 192.168.202.110@tcp last attempt 7614
00000400:00000200:2.0:1786177391.096810:0:84480:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177391.096818:0:84480:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177391.096821:0:84480:0:(import.c:589:import_select_connection()) MGC192.168.202.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177391.096825:0:84480:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.096829:0:84480:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177391.096833:0:84480:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.096835:0:84480:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177391.096840:0:84480:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177391.096843:0:84480:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf73d5
00000020:00000001:2.0:1786177391.096846:0:84480:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177391.096850:0:84480:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1907a6800 refcount=3
00000020:00000001:2.0:1786177391.096853:0:84480:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000020:00000001:2.0:1786177391.096856:0:84480:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525060741120 : -114548648810496 : ffff97d1907a6800)
00000100:00000040:2.0:1786177391.096859:0:84480:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.096861:0:84480:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177391.096863:0:84480:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.096865:0:84480:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177391.096868:0:84480:0:(genops.c:900:class_export_put()) PUTting export ffff97d1907a6800 : new refcount 2
00000100:00080000:2.0:1786177391.096872:0:84480:0:(import.c:624:import_select_connection()) MGC192.168.202.110@tcp: import ffff97d1907a7000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177391.096875:0:84480:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177391.096879:0:84480:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177391.096883:0:84480:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177391.096886:0:84480:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.096948:0:84480:0:(obd_class.h:889:obd_reconnect()) Process entered
00000100:00000001:2.0:1786177391.096952:0:84480:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177391.096964:0:84480:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edd880.
00000020:00000040:2.0:1786177391.096969:0:84480:0:(genops.c:1231:class_import_get()) import ffff97d1907a7000 refcount=4 obd=MGC192.168.202.110@tcp
02000000:00000001:2.0:1786177391.096974:0:84480:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177391.096976:0:84480:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.096981:0:84480:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177391.096985:0:84480:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177391.096991:0:84480:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177391.097000:0:84480:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3800.
00000100:00000001:2.0:1786177391.097009:0:84480:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177391.097021:0:84480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177391.097033:0:84480:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177391.097038:0:84480:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177391.097044:0:84480:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a606a53] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.097101:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177391.097106:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177391.097109:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177391.097117:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.097127:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177391.097129:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177391.097134:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177391.097137:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177391.097140:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177391.097143:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856372992:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177391.097148:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177391.097153:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177391.097156:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177391.097159:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177391.097165:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177391.097167:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177391.097173:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177391.097178:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177391.097186:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177391.097195:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856372992, portal 25
00000100:00000001:1.0:1786177391.097198:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177391.097200:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454182528 : -114548255369088 : ffff97d1a7edd880)
00000100:00000040:1.0:1786177391.097207:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.097215:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177391.097219:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177391.097222:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177391.097225:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872934856372992, offset 0
00000400:00000200:1.0:1786177391.097233:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177391.097244:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177391.097252:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177391.097258:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177391.097263:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177391.097268:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177391.097271:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177391.097275:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255085:255085:256:4294967295] 192.168.202.110@tcp LPNI seq info [255074:255074:8:4294967295]
00000400:00000200:1.0:1786177391.097280:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177391.097291:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177391.097298:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177391.097304:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177391.097313:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177391.097317:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786177391.097323:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177391.097327:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1786177391.097333:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177391.097336:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177391.097345:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177391.097350:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177391.097357:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177391.097359:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177391.097365:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177391.097369:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177391.097398:0:84480:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177391.097402:0:84480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177391.097406:0:84480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1786177391.097408:0:84480:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177391.097410:0:84480:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177391.097413:0:84480:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177391.097416:0:84480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7614
00000400:00000200:2.0:1786177391.097421:0:84480:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177391.097424:0:84480:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177391.097427:0:84480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177391.097430:0:84480:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.097432:0:84480:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177391.097434:0:84480:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.097436:0:84480:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177391.097439:0:84480:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177391.097440:0:84480:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7429
00000020:00000001:2.0:1786177391.097442:0:84480:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177391.097444:0:84480:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a80000 refcount=5
00000020:00000001:2.0:1786177391.097447:0:84480:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000020:00000001:2.0:1786177391.097449:0:84480:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483159552 : -114548226392064 : ffff97d1a9a80000)
00000100:00000040:2.0:1786177391.097453:0:84480:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.097454:0:84480:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177391.097457:0:84480:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.097459:0:84480:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177391.097462:0:84480:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a80000 : new refcount 4
00000100:00080000:2.0:1786177391.097464:0:84480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff97d1885e1000: import ffff97d1a9a80800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177391.097467:0:84480:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177391.097469:0:84480:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177391.097476:0:84480:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177391.097478:0:84480:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.097480:0:84480:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177391.097498:0:84480:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177391.097502:0:84480:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.097504:0:84480:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177391.097508:0:84480:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7ede680.
00000020:00000040:2.0:1786177391.097536:0:84480:0:(genops.c:1231:class_import_get()) import ffff97d1a9a80800 refcount=9 obd=lustre-MDT0000-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177391.097539:0:84480:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177391.097540:0:84480:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.097542:0:84480:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177391.097544:0:84480:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177391.097546:0:84480:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177391.097553:0:84480:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0000.
00000100:00000001:2.0:1786177391.097557:0:84480:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177391.097562:0:84480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177391.097571:0:84480:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177391.097573:0:84480:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177391.097577:0:84480:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c9a13d3e] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.097643:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177391.097649:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.097658:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.097660:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177391.097670:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177391.097675:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177391.097678:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 0 is 4302290638
00000100:00000010:1.0:1786177391.097683:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1360.
00000100:00000001:1.0:1786177391.097691:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 0
00000100:00000001:1.0:1786177391.097703:0:2391:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS new deadline estimation for cpu 1 is 4302290638
00000100:00000010:1.0:1786177391.097707:0:2391:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff97d1a79b1540.
00000100:00000001:1.0:1786177391.097711:0:2391:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.110@tcp: PM QoS request now active for cpu 1
00000100:00000001:1.0:1786177391.097715:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177391.097718:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.097721:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177391.097726:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177391/real 1786177391]  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177391.097737:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177391.097739:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177391.097741:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177391.097744:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177391.097746:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177391.097749:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177391.097753:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177391.097762:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.097769:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177391.097771:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.097774:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177391.097776:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177391.097781:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.097789:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177391.097793:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.097801:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177391.097805:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.097807:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177391.097808:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.097811:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177391.097814:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177391.097817:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1907a7000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177391.097821:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177391.097824:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177391.097828:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177391.097837:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:375c59b7-3e87-4aac-b0e2-3bc92f101731:2391:1872934856372992:192.168.202.110@tcp:250:kworker.0
00000100:00000001:1.0:1786177391.097843:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.097846:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177391.097849:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edd880 x1872934856372992/t0(0) o250->MGC192.168.202.110@tcp@192.168.202.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.097857:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177391.097860:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177391.097862:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177391.097865:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177391.097867:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177391.097870:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1907a7000 refcount=3 obd=MGC192.168.202.110@tcp
00000020:00000001:1.0:1786177391.097872:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177391.097875:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3800.
02000000:00000001:1.0:1786177391.097880:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177391.097882:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177391.097884:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edd880.
00000100:00000001:1.0:1786177391.097887:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177391.097900:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.097903:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.097913:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177391.097915:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.097917:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177391.097919:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177391.097921:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177391.097925:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.097932:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177391.097934:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177391.097936:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177391.097938:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177391.097940:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177391.097943:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856373120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177391.097946:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177391.097949:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177391.097950:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177391.097952:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177391.097955:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177391.097957:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177391.097961:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177391.097964:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177391.097967:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177391.097971:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856373120, portal 10
00000100:00000001:1.0:1786177391.097973:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177391.097974:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454186112 : -114548255365504 : ffff97d1a7ede680)
00000100:00000040:1.0:1786177391.097981:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.097988:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177391.097990:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177391.097993:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177391.097995:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856373120, offset 0
00000400:00000200:1.0:1786177391.097999:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177391.098003:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177391.098008:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177391.098011:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177391.098014:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177391.098017:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177391.098019:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177391.098022:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255086:255086:256:4294967295] 192.168.202.110@tcp LPNI seq info [255075:255075:8:4294967295]
00000400:00000200:1.0:1786177391.098027:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177391.098033:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177391.098038:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177391.098042:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177391.098046:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177391.098048:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786177391.098053:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177391.098056:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1786177391.098059:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177391.098061:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177391.098064:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177391.098068:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177391.098071:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177391.098073:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177391.098076:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177391.098080:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.098088:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177391.098091:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.098098:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.098100:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177391.098103:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177391.098108:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: using a timeout of 8053 usecs (9 jiffies)
00000400:00000200:1.0:1786177391.098111:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177391.098113:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302286858
00000100:00000001:1.0:1786177391.098117:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302290638
00000100:00000001:1.0:1786177391.098119:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302286858
00000100:00000001:1.0:1786177391.098122:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302290638
00000100:00000001:1.0:1786177391.098124:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177391.098126:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.098128:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177391.098132:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177391/real 1786177391]  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177391.098140:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177391.098142:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177391.098143:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177391.098146:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177391.098148:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177391.098150:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177391.098153:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177391.098162:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.098170:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177391.098171:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.098173:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177391.098175:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177391.098179:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.098186:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177391.098190:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.098198:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177391.098201:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.098203:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177391.098205:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.098207:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177391.098209:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177391.098212:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a80800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177391.098215:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177391.098218:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177391.098222:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177391.098231:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856373120:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177391.098235:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.098237:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177391.098240:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7ede680 x1872934856373120/t0(0) o38->lustre-MDT0000-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.098247:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177391.098249:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177391.098251:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177391.098253:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177391.098254:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177391.098256:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a80800 refcount=8 obd=lustre-MDT0000-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177391.098258:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177391.098259:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0000.
02000000:00000001:1.0:1786177391.098263:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177391.098265:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177391.098266:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7ede680.
00000100:00000001:1.0:1786177391.098269:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177391.098270:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.098272:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.098276:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177391.098277:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.098279:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177391.098281:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.098993:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177391.098997:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.099044:0:84480:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177391.099049:0:84480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177391.099055:0:84480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1786177391.099057:0:84480:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177391.099060:0:84480:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177391.099064:0:84480:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177391.099068:0:84480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7614
00000400:00000200:2.0:1786177391.099074:0:84480:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177391.099078:0:84480:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177391.099080:0:84480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177391.099084:0:84480:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099086:0:84480:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177391.099089:0:84480:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099092:0:84480:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177391.099094:0:84480:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177391.099096:0:84480:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7430
00000020:00000001:2.0:1786177391.099098:0:84480:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177391.099101:0:84480:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d1a9a84800 refcount=5
00000020:00000001:2.0:1786177391.099103:0:84480:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000020:00000001:2.0:1786177391.099106:0:84480:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525483177984 : -114548226373632 : ffff97d1a9a84800)
00000100:00000040:2.0:1786177391.099109:0:84480:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099111:0:84480:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177391.099113:0:84480:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099115:0:84480:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177391.099118:0:84480:0:(genops.c:900:class_export_put()) PUTting export ffff97d1a9a84800 : new refcount 4
00000100:00080000:2.0:1786177391.099122:0:84480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff97d1885e1000: import ffff97d1a9a85000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099125:0:84480:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177391.099127:0:84480:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177391.099132:0:84480:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177391.099134:0:84480:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.099137:0:84480:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177391.099140:0:84480:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1786177391.099143:0:84480:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.099145:0:84480:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177391.099150:0:84480:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc000.
00000020:00000040:2.0:1786177391.099154:0:84480:0:(genops.c:1231:class_import_get()) import ffff97d1a9a85000 refcount=6 obd=lustre-MDT0001-mdc-ffff97d1885e1000
02000000:00000001:2.0:1786177391.099157:0:84480:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177391.099159:0:84480:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.099162:0:84480:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177391.099164:0:84480:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177391.099166:0:84480:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177391.099172:0:84480:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3000.
00000100:00000001:2.0:1786177391.099178:0:84480:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177391.099184:0:84480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177391.099193:0:84480:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177391.099196:0:84480:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177391.099201:0:84480:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000145f4fbd] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.099243:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177391.099246:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177391.099248:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177391.099254:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.099262:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177391.099264:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177391.099267:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177391.099268:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177391.099271:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177391.099275:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856373248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177391.099279:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177391.099281:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177391.099284:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177391.099286:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177391.099289:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177391.099292:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177391.099296:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177391.099299:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177391.099303:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177391.099307:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856373248, portal 10
00000100:00000001:1.0:1786177391.099310:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177391.099312:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454176256 : -114548255375360 : ffff97d1a7edc000)
00000100:00000040:1.0:1786177391.099318:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.099325:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177391.099328:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177391.099330:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177391.099333:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872934856373248, offset 0
00000400:00000200:1.0:1786177391.099337:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177391.099343:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177391.099347:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177391.099351:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177391.099355:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177391.099358:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177391.099361:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177391.099364:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255087:255087:256:4294967295] 192.168.202.110@tcp LPNI seq info [255076:255076:8:4294967295]
00000400:00000200:1.0:1786177391.099369:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177391.099377:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177391.099383:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177391.099387:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177391.099391:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177391.099394:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786177391.099399:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177391.099402:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1786177391.099407:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177391.099409:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177391.099413:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177391.099417:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177391.099421:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177391.099423:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177391.099426:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177391.099430:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177391.099456:0:84480:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177391.099461:0:84480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177391.099465:0:84480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery
00000100:00000001:2.0:1786177391.099467:0:84480:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177391.099469:0:84480:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177391.099472:0:84480:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177391.099475:0:84480:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7614
00000400:00000200:2.0:1786177391.099479:0:84480:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177391.099483:0:84480:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177391.099710:0:84480:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177391.099714:0:84480:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099716:0:84480:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177391.099719:0:84480:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099722:0:84480:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177391.099724:0:84480:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177391.099726:0:84480:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf743e
00000020:00000001:2.0:1786177391.099728:0:84480:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177391.099731:0:84480:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d183dca000 refcount=4
00000020:00000001:2.0:1786177391.099733:0:84480:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000020:00000001:2.0:1786177391.099736:0:84480:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629524849074176 : -114548860477440 : ffff97d183dca000)
00000100:00000040:2.0:1786177391.099739:0:84480:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099741:0:84480:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177391.099743:0:84480:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099746:0:84480:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177391.099748:0:84480:0:(genops.c:900:class_export_put()) PUTting export ffff97d183dca000 : new refcount 3
00000100:00080000:2.0:1786177391.099751:0:84480:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff97d1885e1000: import ffff97d1a9a84000 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099754:0:84480:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177391.099756:0:84480:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177391.099759:0:84480:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177391.099761:0:84480:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.099763:0:84480:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177391.099766:0:84480:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177391.099769:0:84480:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.099771:0:84480:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177391.099775:0:84480:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edc380.
00000020:00000040:2.0:1786177391.099778:0:84480:0:(genops.c:1231:class_import_get()) import ffff97d1a9a84000 refcount=5 obd=lustre-OST0000-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177391.099780:0:84480:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177391.099782:0:84480:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.099783:0:84480:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177391.099785:0:84480:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177391.099787:0:84480:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177391.099791:0:84480:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab3400.
00000100:00000001:2.0:1786177391.099795:0:84480:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177391.099800:0:84480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177391.099815:0:84480:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177391.099817:0:84480:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177391.099822:0:84480:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000266c4382] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177391.099830:0:84480:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1786177391.099834:0:84480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 140a5896-206d-4996-9565-aa0967196f03->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1786177391.099838:0:84480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1786177391.099840:0:84480:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1786177391.099842:0:84480:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1786177391.099845:0:84480:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1786177391.099847:0:84480:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: connect to NID 192.168.202.110@tcp last attempt 7614
00000400:00000200:2.0:1786177391.099852:0:84480:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:2.0:1786177391.099855:0:84480:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1786177391.099857:0:84480:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1786177391.099860:0:84480:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099863:0:84480:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177391.099865:0:84480:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099867:0:84480:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000001:2.0:1786177391.099869:0:84480:0:(genops.c:813:class_conn2export()) Process entered
00000020:00000040:2.0:1786177391.099870:0:84480:0:(genops.c:825:class_conn2export()) looking for export cookie 0xdd5d0a6aadcf7445
00000020:00000001:2.0:1786177391.099872:0:84480:0:(lustre_handles.c:129:class_handle2object()) Process entered
00000020:00000040:2.0:1786177391.099875:0:84480:0:(lustre_handles.c:147:class_handle2object()) GET export ffff97d19a15d000 refcount=4
00000020:00000001:2.0:1786177391.099877:0:84480:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000020:00000001:2.0:1786177391.099879:0:84480:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446629525221920768 : -114548487630848 : ffff97d19a15d000)
00000100:00000040:2.0:1786177391.099882:0:84480:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff97d1831ead80 refcount 14 to 192.168.202.110@tcp
00000100:00000001:2.0:1786177391.099884:0:84480:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1786177391.099886:0:84480:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff97d1831ead80 refcount 15 to 192.168.202.110@tcp
00000100:00000001:1.0:1786177391.099916:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177391.099921:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.099930:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.099932:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177391.099938:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177391.099943:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: using a timeout of 133081 usecs (134 jiffies)
00000400:00000200:1.0:1786177391.099946:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177391.099949:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302286985
00000100:00000001:1.0:1786177391.099952:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302290638
00000100:00000001:1.0:1786177391.099955:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302286985
00000100:00000001:1.0:1786177391.099958:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302290638
00000100:00000001:1.0:1786177391.099960:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177391.099963:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.099965:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177391.099970:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177391/real 1786177391]  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177391.099980:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177391.099982:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177391.099984:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177391.099987:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177391.099989:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177391.099992:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177391.099995:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177391.100004:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100012:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177391.100013:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.100016:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177391.100018:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177391.100022:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100029:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177391.100034:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100042:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177391.100045:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.100046:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177391.100049:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.100051:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177391.100054:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177391.100057:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a85000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177391.100061:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177391.100063:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177391.100068:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177391.100077:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856373248:192.168.202.110@tcp:38:kworker.0
00000100:00000001:1.0:1786177391.100082:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.100085:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177391.100088:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc000 x1872934856373248/t0(0) o38->lustre-MDT0001-mdc-ffff97d1885e1000@192.168.202.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100096:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177391.100098:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177391.100101:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177391.100104:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177391.100106:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177391.100108:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a85000 refcount=5 obd=lustre-MDT0001-mdc-ffff97d1885e1000
00000020:00000001:1.0:1786177391.100110:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177391.100113:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3000.
02000000:00000001:1.0:1786177391.100116:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177391.100117:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177391.100119:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc000.
00000100:00000001:1.0:1786177391.100122:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177391.100123:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.100125:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.100130:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177391.100132:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.100134:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177391.100136:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177391.100138:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177391.100142:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100149:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177391.100150:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177391.100152:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177391.100154:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177391.100156:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177391.100159:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856373376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177391.100162:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177391.100164:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177391.100166:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177391.100168:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177391.100171:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177391.100173:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177391.100176:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177391.100179:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177391.100182:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177391.100186:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856373376, portal 4
00000100:00000001:1.0:1786177391.100188:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177391.100189:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454177152 : -114548255374464 : ffff97d1a7edc380)
00000100:00000040:1.0:1786177391.100196:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100202:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177391.100205:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177391.100207:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177391.100210:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856373376, offset 0
00000400:00000200:1.0:1786177391.100214:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177391.100218:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177391.100222:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177391.100225:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177391.100228:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177391.100231:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177391.100233:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177391.100237:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255088:255088:256:4294967295] 192.168.202.110@tcp LPNI seq info [255077:255077:8:4294967295]
00000400:00000200:1.0:1786177391.100242:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177391.100248:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177391.100253:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177391.100257:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177391.100261:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177391.100264:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786177391.100269:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177391.100271:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1786177391.100275:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177391.100277:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177391.100281:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177391.100284:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177391.100287:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177391.100289:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177391.100292:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177391.100295:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100303:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177391.100307:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100314:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.100315:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177391.100318:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177391.100322:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: using a timeout of 8728 usecs (9 jiffies)
00000400:00000200:1.0:1786177391.100325:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177391.100327:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302286860
00000100:00000001:1.0:1786177391.100330:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302290638
00000100:00000001:1.0:1786177391.100332:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302286860
00000100:00000001:1.0:1786177391.100335:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302290638
00000100:00000001:1.0:1786177391.100337:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177391.100340:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.100342:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177391.100345:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177391/real 1786177391]  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177391.100353:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177391.100354:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177391.100356:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177391.100359:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177391.100361:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177391.100363:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177391.100366:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177391.100374:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100381:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177391.100383:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.100385:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177391.100386:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177391.100390:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100397:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177391.100401:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100409:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177391.100411:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.100413:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177391.100414:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.100416:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177391.100418:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177391.100421:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a84000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177391.100424:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177391.100427:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177391.100431:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177391.100439:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856373376:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177391.100443:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.100445:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177391.100448:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edc380 x1872934856373376/t0(0) o8->lustre-OST0000-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100455:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177391.100456:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177391.100458:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177391.100461:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177391.100463:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177391.100464:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a84000 refcount=4 obd=lustre-OST0000-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177391.100466:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177391.100468:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab3400.
02000000:00000001:1.0:1786177391.100470:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177391.100471:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177391.100473:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edc380.
00000100:00000001:1.0:1786177391.100475:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177391.100476:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.100478:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.100482:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177391.100483:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.100654:0:84480:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629524836625792 : -114548872925824 : ffff97d1831ead80)
00000020:00000040:2.0:1786177391.100659:0:84480:0:(genops.c:900:class_export_put()) PUTting export ffff97d19a15d000 : new refcount 3
00000100:00080000:2.0:1786177391.100663:0:84480:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff97d1885e1000: import ffff97d1a9a82800 using connection 192.168.202.110@tcp/192.168.202.110@tcp
00000100:00000001:2.0:1786177391.100666:0:84480:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1786177391.100668:0:84480:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1786177391.100671:0:84480:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1786177391.100674:0:84480:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.100676:0:84480:0:(obd_class.h:889:obd_reconnect()) Process entered
00000008:00100000:2.0:1786177391.100678:0:84480:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1786177391.100681:0:84480:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.100683:0:84480:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1786177391.100687:0:84480:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff97d1a7edca80.
00000020:00000040:2.0:1786177391.100691:0:84480:0:(genops.c:1231:class_import_get()) import ffff97d1a9a82800 refcount=4 obd=lustre-OST0001-osc-ffff97d1885e1000
02000000:00000001:2.0:1786177391.100693:0:84480:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1786177391.100695:0:84480:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1786177391.100697:0:84480:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1786177391.100699:0:84480:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1786177391.100701:0:84480:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1786177391.100706:0:84480:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff97d186ab0800.
00000100:00000001:2.0:1786177391.100710:0:84480:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1786177391.100715:0:84480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1786177391.100723:0:84480:0:(jobid.c:936:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1786177391.100726:0:84480:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177391.100730:0:84480:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003b76d4ef] to pc [ptlrpcd_rcv+-1]  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1786177391.100738:0:84480:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1786177391.100744:0:84480:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (7624)
00000100:00000001:1.0:1786177391.100808:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177391.100810:0:2391:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1786177391.100812:0:2391:0:(client.c:1800:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1786177391.100816:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100825:0:2391:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1786177391.100826:0:2391:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177391.100829:0:2391:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1786177391.100831:0:2391:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1786177391.100833:0:2391:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1786177391.100836:0:2391:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856373504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177391.100840:0:2391:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1786177391.100842:0:2391:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1786177391.100844:0:2391:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1786177391.100845:0:2391:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1786177391.100849:0:2391:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177391.100851:0:2391:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1786177391.100855:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000010:1.0:1786177391.100858:0:2391:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff97d189b8dbb0.
00000400:00000010:1.0:1786177391.100861:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c770.
00000100:00000200:1.0:1786177391.100865:0:2391:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872934856373504, portal 4
00000100:00000001:1.0:1786177391.100867:0:2391:0:(client.c:3262:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1786177391.100868:0:2391:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446629525454178944 : -114548255372672 : ffff97d1a7edca80)
00000100:00000040:1.0:1786177391.100875:0:2391:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100882:0:2391:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1786177391.100885:0:2391:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.110@tcp
00000400:00000010:1.0:1786177391.100887:0:2391:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff97d189b8c660.
00000100:00000200:1.0:1786177391.100902:0:2391:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872934856373504, offset 0
00000400:00000200:1.0:1786177391.100907:0:2391:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.110@tcp
00000400:00000200:1.0:1786177391.100912:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177391.100916:0:2391:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.202.110@tcp local destination
00000400:00000200:1.0:1786177391.100920:0:2391:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1786177391.100922:0:2391:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.10@tcp
00000400:00000200:1.0:1786177391.100926:0:2391:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1786177391.100928:0:2391:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.110@tcp
00000400:00000200:1.0:1786177391.100931:0:2391:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.10@tcp NI seq info: [255089:255089:256:4294967295] 192.168.202.110@tcp LPNI seq info [255078:255078:8:4294967295]
00000400:00000200:1.0:1786177391.100936:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000400:00000200:1.0:1786177391.100943:0:2391:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.10@tcp(192.168.202.10@tcp:<?>) -> 192.168.202.110@tcp(192.168.202.110@tcp:192.168.202.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1786177391.100948:0:2391:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177391.100951:0:2391:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff97d19881ad00.
00000800:00000200:1.0:1786177391.100955:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.0:1786177391.100958:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1786177391.100963:0:2391:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff97d18514da80] -> 12345-192.168.202.110@tcp (2)
00000800:00000200:1.1:1786177391.100965:0:2391:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1786177391.100969:0:2391:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.110@tcp
00000800:00000010:1.0:1786177391.100971:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff97d19881ad00.
00000400:00000200:1.0:1786177391.100974:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1786177391.100978:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.10@tcp->192.168.202.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1786177391.100981:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c660
00000400:00000010:1.0:1786177391.100983:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c660.
00000100:00000001:1.0:1786177391.100986:0:2391:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1786177391.100990:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.100998:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177391.101001:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177416 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.101008:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.101010:0:2391:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1786177391.101013:0:2391:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1786177391.101016:0:2391:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: using a timeout of 16751 usecs (17 jiffies)
00000400:00000200:1.0:1786177391.101020:0:2391:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.110@tcp to cpt 0
00000100:00000001:1.0:1786177391.101022:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 0 is 4302286869
00000100:00000001:1.0:1786177391.101025:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 0, keep current deadline 4302290638
00000100:00000001:1.0:1786177391.101027:0:2391:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS new deadline estimation for cpu 1 is 4302286869
00000100:00000001:1.0:1786177391.101029:0:2391:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff97d1885e1000: PM QoS request active for cpu 1, keep current deadline 4302290638
00000100:00000001:1.0:1786177391.101032:0:2391:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1786177391.101034:0:2391:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.101035:0:2391:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1786177391.101039:0:2391:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177391/real 1786177391]  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1786177391.101048:0:2391:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1786177391.101049:0:2391:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1786177391.101051:0:2391:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff97d189b8dbb0.
00000400:00000200:1.0:1786177391.101053:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff97d189b8c770
00000400:00000010:1.0:1786177391.101055:0:2391:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff97d189b8c770.
00000100:00000001:1.0:1786177391.101057:0:2391:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1786177391.101060:0:2391:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1786177391.101069:0:2391:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.101076:0:2391:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1786177391.101077:0:2391:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.101080:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177391.101081:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1786177391.101085:0:2391:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.101092:0:2391:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1786177391.101096:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.101103:0:2391:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1786177391.101106:0:2391:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.101108:0:2391:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1786177391.101110:0:2391:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.101112:0:2391:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1786177391.101113:0:2391:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1786177391.101116:0:2391:0:(import.c:65:import_set_state_nolock()) ffff97d1a9a82800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1786177391.101120:0:2391:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.110@tcp failed (-110)
00000100:00000001:1.0:1786177391.101122:0:2391:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1786177391.101127:0:2391:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1786177391.101136:0:2391:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff97d1a7edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:140a5896-206d-4996-9565-aa0967196f03:2391:1872934856373504:192.168.202.110@tcp:8:kworker.0
00000100:00000001:1.0:1786177391.101140:0:2391:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.101142:0:2391:0:(client.c:2854:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1786177391.101145:0:2391:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff97d1a7edca80 x1872934856373504/t0(0) o8->lustre-OST0001-osc-ffff97d1885e1000@192.168.202.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177416 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1786177391.101152:0:2391:0:(client.c:2780:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1786177391.101153:0:2391:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1786177391.101155:0:2391:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff97d186b00800.
02000000:00000001:1.0:1786177391.101158:0:2391:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1786177391.101159:0:2391:0:(genops.c:1238:class_import_put()) Process entered
00000020:00000040:1.0:1786177391.101160:0:2391:0:(genops.c:1244:class_import_put()) import ffff97d1a9a82800 refcount=3 obd=lustre-OST0001-osc-ffff97d1885e1000
00000020:00000001:1.0:1786177391.101162:0:2391:0:(genops.c:1251:class_import_put()) Process leaving
02000000:00000010:1.0:1786177391.101164:0:2391:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff97d186ab0800.
02000000:00000001:1.0:1786177391.101167:0:2391:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1786177391.101169:0:2391:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1786177391.101170:0:2391:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff97d1a7edca80.
00000100:00000001:1.0:1786177391.101173:0:2391:0:(client.c:2836:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1786177391.101174:0:2391:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.101176:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1786177391.101180:0:2391:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1786177391.101181:0:2391:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1786177391.101183:0:2391:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1786177391.101185:0:2391:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1786177391.288672:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177391.288683:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177391.288687:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177391.288695:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177391.288700:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177391.288704:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177391.288707:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177391.288711:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177391.288714:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177391.672680:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177391.672691:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177391.672694:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177391.672698:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177391.672703:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177391.672706:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177391.672709:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177391.672711:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177391.672716:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177391.672718:0:104921:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:1.0:1786177391.672720:0:104921:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:1.0:1786177391.672721:0:104921:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177391.672723:0:104921:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:1.0:1786177391.672724:0:104921:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177391.672726:0:104921:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177391.672728:0:104921:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:1.0:1786177391.672731:0:104921:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:1.0:1786177391.672733:0:104921:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1786177391.672858:0:92300:0:(genops.c:1916:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1786177391.672862:0:92300:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1786177391.672865:0:92300:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1786177392.312590:0:104399:0:(client.c:2554:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1786177392.312601:0:104399:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1786177392.312603:0:104399:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1786177392.312610:0:104399:0:(client.c:2702:ptlrpc_set_wait()) set ffff97d1982b53c0 going to sleep for 0 seconds
00000100:00000001:3.0:1786177392.312615:0:104399:0:(client.c:1977:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1786177392.312619:0:104399:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177392.312622:0:104399:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1786177392.312626:0:104399:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1786177392.312629:0:104399:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
Debug log: 134812 lines, 134812 kept, 0 dropped, 0 bad.